diff --git a/build.zig b/build.zig index edebcb739c..7f8a6d25cd 100644 --- a/build.zig +++ b/build.zig @@ -96,6 +96,7 @@ pub fn build(b: *std.Build) void { @import("zig/build/digest.zig").steps(b, test_step, draft_ids, build_options); @import("zig/build/flashnext_budget.zig").steps(b, test_step); @import("zig/build/flashnext_batch_host.zig").tests(b, test_step); + @import("zig/build/lfm3_host.zig").steps(b, test_step); if (b.graph.host.result.os.tag == .macos) { const budget_mods = metalEngineModules(b, b.graph.host, .Debug, draft_ids); const budget_empty = b.addWriteFiles().add("budget-compile-only.metallib", ""); diff --git a/docs/receipts/lfm3/README.md b/docs/receipts/lfm3/README.md new file mode 100644 index 0000000000..4d7d5d089d --- /dev/null +++ b/docs/receipts/lfm3/README.md @@ -0,0 +1,188 @@ +# LFM3 Metal qualification receipts + +These receipts cover native LFM3-1.7B and LFM3-VL-2B from the official MLX exports. +They distinguish implementation checks from the still-required maintainer model/performance gate. +Checkpoint revisions and run instructions are in the [family recipe](../../recipes/lfm3-native.md). + +## Builds and hardware + +- Initial corrected native: `a8df9730ee637237545311a6753dd4bdcb13e860700d65a01a033ae818ee0d18`. +- Optimized benchmark native: `5c8a04d4af599879797f4f5bed8e00ab7d9d70732e698bd2a1ec2a01ec762e09`. +- Guarded row-pair build: `251079309919aa9b21ca8cc6a89c58fc3fc9a9baf800fa5ce6a6af313a5b5f65`. + +The guarded row-pair build additionally closes a diagnostic encoder when counter setup fails. +That change does not alter the production projection or attention arithmetic. Its serving checks were rerun separately. + +Correctness runs used an M4 Pro with 48 GB RAM, macOS 26.5.1 and Zig 0.17.0. +Controlled performance runs used an M5 Ultra with 256 GiB RAM and macOS 27.0 (26A428). +A measured footprint on the M5 is not a smaller-RAM qualification. + +## Served and operator checks + +The optimized M4 binary passed all six greedy/seeded-sampling suites: 303 passed, zero failed or skipped. +These cover drafted/plain, concurrent/solo, real cache reuse, edited prefixes, image order/content, +corrupt input and cancellation/recovery. The guarded row-pair binary also passed 303/303 on the M5. + +Text4 and VL4 grammar runs on both hosts each passed 32 configurations, concurrent comparisons and five refusals. +On each host, full-token capture verified 64/64 configuration groups, including hidden reasoning, across drafted/plain/concurrent requests. +An additional 80 checks covered real required/named tools, typed arguments, streaming, tool-result history and image-conditioned calls. +Grammar requires the matching optional library; these tests used an isolated bundle with library SHA-256 +`04133accdcecd01cd7dd4e8730855905850967ed8c6bd6d8a637258264ae7687`. + +The 4-bit text long suite passed 73/73, including chunk and 8,193-token seams. +The row-pair LFM3-only contract run on the M4 reported: + +```text +SUMMARY 50 kernels: 50 pass gates 1-4, 0 fail, 0 known findings; 13 controls: 13 red as planted, 0 missed; 0 limit problems; gates 19521 ms, limits 133 ms +``` + +The full earlier M5 runner reported 272/273 kernels passing, two known findings, all 298 controls red and zero limit problems. +Its Qwen4 HC activation failure reproduces exactly in a separate build of pinned base `b30d5b9d3`. +That is not a global pass or a final-source global-suite receipt. + +## Shared-stream attention follow-up + +`shared-attention.json` records the subsequent shared-dispatch source and binary identity. +Full and sliding attention reuse the serial partial and merge arithmetic, with stream-local cache addresses and row maps. +All 52 LFM3 kernels passed gates 1–4, all 14 controls were red as planted, and launch limits had zero problems. +The real text4 default exactness suite passed 68/68, including unequal two/four-stream state and logits. +All six sampled/cancellation serving suites on the M5 passed 303/303 with zero failures or skips. +This follow-up is byte-preserving shared scheduling; it does not qualify tensor-unit prompt attention or a speed gate. + +## Numerical evidence + +`projection-fp64-bounds.json` covers 66 projection/layout cells and 60,954,624 outputs. +The actual shared FP32 helper was captured before BF16 storage and compared with independently unpacked affine FP64 products. +No raw FP32 or BF16-store error exceeded the existing derived bounds. Worst-column errors were retained. + +The row-pair optimization invokes the same shared helper and preserves its accumulation order. +All 132 checkpoint comparisons at widths 1 and 64 produced 123,814,080 bytes identical to the old selector. +A separate 8,193-token before/after check compared all 2,097,408,000 BF16 logit bytes exactly. +Thus this optimization does not worsen any output-based precision metric on those comparisons. + +`fidelity-investigation.json` records the first actual primitive difference and its causal propagation. +Official MLX 0.32.4 uses different effective affine reconstruction in its vector and matrix paths. +The native engine consistently follows the FP32 affine equation rather than changing arithmetic with batch width. +The first captured FP32 error was about 6.81e-9, inside the existing derived 5.52e-5 pre-store bound; +it crossed a BF16 midpoint and propagated through recurrent history. + +`model-fp64-comparison.json` retains all 18 originally disjoint native/MLX positions, including repeated histories. +With direct binary64-to-BF16 rounding, native belongs to the ideal model maximum set in nine cases, MLX in 16, +both in eight and neither in one. This ideal FP64 model is a diagnostic; it does not execute the native FP32 reductions. +No new tie tolerance or end-to-end acceptance threshold was applied. The primitive evidence does not replace that model gate. + +The oracle is reproducible with `tools/lfm3/fp64_reference.py`; its tests include all finite BF16 encodings, +adjacent midpoints, double rounding, subnormals, overflow and six-bit cross-word unpacking. + +## Binned projection candidate + +`binned-projection.json` records the explicit candidate at `75478ed17`; ordinary model loading leaves it disabled. +It preserves each input value while partitioning exponent ranges and exact code nibbles. M5 uses the shared +`nax.h` tensor-unit fragments; M4 uses its fallback. Group affine sums use compensated FP32 pairs. +The integer argument is limited to finite normal aligned products without overflow or underflow. +It does not establish universal FP64 equivalence for extreme BF16 subnormals or arbitrary inputs. + +The actual native M5 tool produced **61,907,040/61,907,040** direct FP64-to-BF16 reference values across 132 captures: +11 projection classes, all six checkpoints, and prompt/decode widths 64 and 1. All 66 first-row comparisons matched. +The six small attention-gate cells use the unchanged row fallback and are included explicitly. +No column had more FP64 error than either the retained original-native or MLX result. +Those two comparator captures are from the M4, not a same-device M5 performance comparison. +The file retains worst-column metrics, input/output hashes, artifact identities and the complete contract summaries. +Both M4 and M5 passed all six operator variants, both planted controls and launch limits. +These primitive results do not settle whole-model fidelity or served performance. + +To reconstruct the fixtures, save `fixture_tokens` as a little-endian uint32 NPY array and run the default +`tf-lfm3-forward MODEL TOKENS.npy baseline.bf16 trace TRACE_DIR` for each named checkpoint. +Concatenate the trace files ending in `_.bf16` in filename order to recreate each `input_file`; +the first `2*k` bytes recreate its `one_row_file`. Verify the recorded input hash before comparing outputs. +Run `tf-lfm3-project MODEL MODULE INPUT.bf16 ROWS OUTPUT.bf16 --binned` at widths 64 and 1. +Save the receipt’s `cases` array as `manifest.json` beside the inputs for `tools/lfm3/projection_compare.py`. +Use the frozen candidate commit for these artifact identities. + +`binned-row-twin.json` records the later opt-in row twin at `5a2bdea76`: all 132 captures retain +123,814,080 bytes exactly against the earlier FP64-exact tiled captures. Both M4 and M5 pass six +variants and two planted controls. The twin remains disabled for ordinary model loading because +its serving performance has not passed; these operator results do not qualify a whole model. + +## Controlled performance + +`cold-prefill-ab.json` contains all 16 matched pairs / 32 fresh-server cold requests. +It includes full output-token hashes, body hashes, client TTFT, engine prefill time and peak physical process footprint. +Both sides used the same body, exact prompt counts, 16 output tokens, context 70,000, four configured lanes and zero cached tokens. +The 2k and 8k cells have three alternating pairs each; 32k and 64k have one pair each. +Only 4-bit performance was measured. + +Engine prefill seconds, before to after: + +- Text: 2k 1.239530 to 0.812959; 8k 5.307524 to 3.608895; 32k 27.314691 to 20.488389; 64k 73.402432 to 61.632933. +- VL decoder, text requests: 2k 1.261845 to 0.815432; 8k 5.402433 to 3.617449; 32k 27.686643 to 20.536702; 64k 73.920498 to 59.646930. + +These are engine prefill times, not client TTFT. All 16 paired full output-token hashes matched. +At 64k, physical footprint was 2,926,167,768 to 2,925,315,776 bytes for text and +3,586,115,720 to 3,584,559,264 bytes for VL. Device/pinned allocator peaks were not collected. + +`decode-concurrency.json` records 256-token, three-repetition code/prose runs. +All 60 concurrent-to-solo comparisons and all eight serial checks matched. +Small decode-rate differences are descriptive; they are not a separately established speedup. + +Earlier standard-server comparisons still showed MLX-VLM faster, especially for long cold prefill. +The faster arithmetic-changing matrix candidate failed the strict no-worse precision comparison and was excluded. +The slower uniform-exponential candidate was also excluded. No unvalidated GQA experiment is enabled. + +## Paired FFN and CPU greedy follow-up + +`greedy-dual.json` records the subsequent immutable `b804c6ed` native build. +The FFN gate/up projections share one dispatch while retaining each projection's shared helper and BF16 stores. +A SIMD CPU greedy reduction preserves the first maximum, signed-zero ties and nonfinite rejection; +masked and positive-temperature sampling keep the original path. +All 58 LFM3 kernel checks passed, all 16 controls were red, and launch limits had zero problems. +The real text4 exactness suite passed 68/68; six M5 sampled/cancellation serving suites passed 303/303. +Actual projection pairs retained 247,628,160 bytes, and six complete teacher-forced logit arrays retained 394,752,000 bytes. + +Matched M5 old/new/new/old runs used three repetitions and 256 output tokens per request. +All 240 concurrent full-token digests matched across the corresponding prompts and legs; +240/240 concurrent-to-solo and 32/32 serial checks passed. +Text code/prose single-stream rates changed from 309.27/322.59 to 317.39/332.83 tok/s; +four-stream aggregate rates changed from 682.50/717.25 to 836.65/856.70 tok/s. +VL decoder code/prose rates changed from 292.69/290.90 to 301.68/299.90 tok/s; +four-stream rates changed from 656.00/658.15 to 813.40/810.10 tok/s. +These are medians of the two run medians, with all raw three-repetition cells retained. +This ABBA experiment measures only 4-bit performance and does not assert standard-server parity. + +## Reproduce + +Use the pinned Zig version and a complete local checkpoint. Developer references need NumPy; the MLX reference also needs MLX. + +```sh +zig build native tf-lfm3-forward tf-lfm3-project tf-lfm3-profile tf-lfm3-exact tf-kernel-contract -Dcpu=apple_m1 +zig build test test-lfm3-host test-kernel-contract test-golden fuzz-cortheon -Dcpu=apple_m1 +python3 -m unittest discover -s tools/lfm3 -p 'test_*.py' +./zig-out/bin/tf-kernel-contract --only LFM3 +./zig-out/bin/tf-lfm3-exact MODEL_DIR --long +python3 tools/lfm3/serve_check.py --binary ./zig-out/native/bin/tensorfold-native --model-dir MODEL_DIR --sampled --cancel --output served.json +python3 tools/lfm3/reference.py --model-dir MODEL_DIR --tokens TOKENS.npy --output mlx.bf16 +python3 tools/lfm3/fp64_reference.py --model-dir MODEL_DIR --tokens TOKENS.npy --output fp64.bf16 --weight-mode exact +./zig-out/bin/tf-lfm3-forward MODEL_DIR TOKENS.npy native.bf16 +./zig-out/bin/tf-lfm3-profile MODEL_DIR TOKENS.npy profile.json +``` + +Repeat the served command for all six checkpoints. Run one model process per GPU, using the native lock and admission guards. +`cold-prompts/` contains the exact request bodies and official rendered token arrays; SHA-256 hashes are in `cold-prompt-sha256.json`. +The native server did not return its rendered arrays, so matching counts are not described as a captured native-token identity proof. +To repeat cold HTTP measurements, start a fresh owned server with `--context 70000 --parallel 4 --prompt-cache-gib 0`, +POST a saved body to `/v1/chat/completions`, and retain usage, full token hash and timings from the stream. +The supplied bodies omit the optional model alias. Keep the exact bodies unchanged between compared builds. + +## All-width matched baseline + +`all-width-performance-baseline.json` records the shared-attention binary before the SIMD greedy and paired-FFN follow-ups. +Both native and the standard server completed 72 retained single-stream requests, 360 retained concurrency requests and 48 cold requests in total. +Every measured decode/concurrency reply has 256 tokens; each cold request has 16 output tokens and zero cached prompt tokens. +All 180 native concurrent-to-solo and 24 serial comparisons matched. Exact public prompts, client hashes, token hashes and timings are retained. +There is one cold sample per cell and three decode/concurrency repetitions. These baseline rates remain below the standard server; they are not a performance-gate pass. +The native binary SHA is `d1656249bdb581b147faacbd3b3c557c00b6c45b710eeddce517a67178b6ba2b`. + +The public long-prompt wrapper is `tools/lfm3/bench_decode.py`; its prompts match the recorded JSON exactly. +Use `python3 tools/lfm3/bench_decode.py decode URL MODEL --tokens 256 --reps 3 --temperatures 0 --output decode.json`. +For concurrency use the `concurrent` mode with `--levels 1,4 --alone --serial` on native, or `--levels 1,4 --no-seed` on standard. +Verify every reply length in the evidence; a shorter standard reply does not establish an equal-length rate. diff --git a/docs/receipts/lfm3/all-fp64-top-differences.json b/docs/receipts/lfm3/all-fp64-top-differences.json new file mode 100644 index 0000000000..95cf7bb827 --- /dev/null +++ b/docs/receipts/lfm3/all-fp64-top-differences.json @@ -0,0 +1,4384 @@ +{ + "scope": "Existing post-RoPE baseline prompt captures against direct FP64/BF16 oracle, selected 17 cells previously containing native/MLX disjoint maxima; not final prompt-tile qualification; nominal fixture names can exceed actual encoded row count, which is reported per cell", + "accepted_threshold": null, + "fidelity_pass": null, + "cells": [ + { + "rows": 257, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.0259931724353837, + "maximum_absolute_error": 2.3125, + "differences": [ + { + "row": 58, + "native_top": 3599, + "reference_top": 2292, + "native_maximum_set": [ + 3599 + ], + "reference_maximum_set": [ + 2292, + 3599 + ], + "maximum_sets_overlap": true, + "native_margin": 0.375, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.375, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 106, + "rms_error": 0.3824222834630468, + "maximum_absolute_error": 2.3125 + }, + { + "row": 82, + "rms_error": 0.30074669025232487, + "maximum_absolute_error": 1.609375 + }, + { + "row": 226, + "rms_error": 0.2786312196301807, + "maximum_absolute_error": 1.625 + }, + { + "row": 178, + "rms_error": 0.2060254502346194, + "maximum_absolute_error": 1.1875 + }, + { + "row": 0, + "rms_error": 0.16957791884638174, + "maximum_absolute_error": 0.875 + }, + { + "row": 233, + "rms_error": 0.16576514601328005, + "maximum_absolute_error": 1.0625 + }, + { + "row": 130, + "rms_error": 0.1439680502141286, + "maximum_absolute_error": 0.7265625 + }, + { + "row": 202, + "rms_error": 0.1433408438786218, + "maximum_absolute_error": 0.6875 + }, + { + "row": 154, + "rms_error": 0.13715977800436777, + "maximum_absolute_error": 0.8125 + }, + { + "row": 6, + "rms_error": 0.11535344107408924, + "maximum_absolute_error": 0.53125 + }, + { + "row": 254, + "rms_error": 0.11291650606233904, + "maximum_absolute_error": 0.421875 + }, + { + "row": 34, + "rms_error": 0.10952884106608807, + "maximum_absolute_error": 0.4375 + }, + { + "row": 204, + "rms_error": 0.10401968151545389, + "maximum_absolute_error": 0.34375 + }, + { + "row": 214, + "rms_error": 0.10219409674230935, + "maximum_absolute_error": 0.4375 + }, + { + "row": 168, + "rms_error": 0.10176202008980587, + "maximum_absolute_error": 0.375 + }, + { + "row": 150, + "rms_error": 0.10112804070082612, + "maximum_absolute_error": 0.3798828125 + }, + { + "row": 96, + "rms_error": 0.099484465055359, + "maximum_absolute_error": 0.359375 + }, + { + "row": 107, + "rms_error": 0.09927835658893223, + "maximum_absolute_error": 0.375 + }, + { + "row": 211, + "rms_error": 0.09746816630877914, + "maximum_absolute_error": 0.3125 + }, + { + "row": 77, + "rms_error": 0.09738425343068764, + "maximum_absolute_error": 0.34375 + } + ], + "worst_columns": [ + { + "column": 2789, + "rms_error": 0.19791373342423282, + "maximum_absolute_error": 2.3125 + }, + { + "column": 17913, + "rms_error": 0.18852943222159388, + "maximum_absolute_error": 2.0625 + }, + { + "column": 3008, + "rms_error": 0.17548858648564844, + "maximum_absolute_error": 1.75 + }, + { + "column": 682, + "rms_error": 0.17545829359413284, + "maximum_absolute_error": 2.0625 + }, + { + "column": 24629, + "rms_error": 0.1731859451035362, + "maximum_absolute_error": 1.9140625 + }, + { + "column": 77351, + "rms_error": 0.16903101829303036, + "maximum_absolute_error": 1.84375 + }, + { + "column": 90344, + "rms_error": 0.16862482688719246, + "maximum_absolute_error": 2.07421875 + }, + { + "column": 1286, + "rms_error": 0.1684901555618048, + "maximum_absolute_error": 1.6875 + }, + { + "column": 4538, + "rms_error": 0.16809228819634683, + "maximum_absolute_error": 1.5625 + }, + { + "column": 20121, + "rms_error": 0.16482718581092243, + "maximum_absolute_error": 1.921875 + }, + { + "column": 2292, + "rms_error": 0.16345997021338707, + "maximum_absolute_error": 1.875 + }, + { + "column": 2419, + "rms_error": 0.1618426122355798, + "maximum_absolute_error": 1.4375 + }, + { + "column": 99838, + "rms_error": 0.1613245817053331, + "maximum_absolute_error": 1.3125 + }, + { + "column": 27212, + "rms_error": 0.16094722668692374, + "maximum_absolute_error": 1.6875 + }, + { + "column": 10030, + "rms_error": 0.1599323655202937, + "maximum_absolute_error": 1.453125 + }, + { + "column": 116541, + "rms_error": 0.1583345269459225, + "maximum_absolute_error": 1.734375 + }, + { + "column": 6165, + "rms_error": 0.156473205200127, + "maximum_absolute_error": 1.63671875 + }, + { + "column": 1813, + "rms_error": 0.1556869395639885, + "maximum_absolute_error": 1.75 + }, + { + "column": 111665, + "rms_error": 0.1553878533965645, + "maximum_absolute_error": 1.75390625 + }, + { + "column": 33006, + "rms_error": 0.15496943800268753, + "maximum_absolute_error": 1.66015625 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "1.7B-6-prose-257", + "phase": "prompt", + "native_logits_sha256": "2fb12160d203b843337c07b5a3e5b7a1f35de658857c3e06d3827c77dac1d001", + "reference_logits_sha256": "3f4c9736cbcbbb893519fa7d1e70e3953df7337e09c4d4764a642cc6e116c89f", + "token_fixture_sha256": "522ec0f7a7ef8481d2f5533e3962ddad1f418d4337b095282e40c096e2befb1d" + }, + { + "rows": 481, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.024320309575124863, + "maximum_absolute_error": 2.3125, + "differences": [ + { + "row": 58, + "native_top": 3599, + "reference_top": 2292, + "native_maximum_set": [ + 3599 + ], + "reference_maximum_set": [ + 2292, + 3599 + ], + "maximum_sets_overlap": true, + "native_margin": 0.375, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.375, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 106, + "rms_error": 0.3824222834630468, + "maximum_absolute_error": 2.3125 + }, + { + "row": 82, + "rms_error": 0.30074669025232487, + "maximum_absolute_error": 1.609375 + }, + { + "row": 329, + "rms_error": 0.2954075529262704, + "maximum_absolute_error": 1.2421875 + }, + { + "row": 226, + "rms_error": 0.2786312196301807, + "maximum_absolute_error": 1.625 + }, + { + "row": 257, + "rms_error": 0.21701386139479145, + "maximum_absolute_error": 1.046875 + }, + { + "row": 178, + "rms_error": 0.2060254502346194, + "maximum_absolute_error": 1.1875 + }, + { + "row": 0, + "rms_error": 0.16957791884638174, + "maximum_absolute_error": 0.875 + }, + { + "row": 233, + "rms_error": 0.16576514601328005, + "maximum_absolute_error": 1.0625 + }, + { + "row": 130, + "rms_error": 0.1439680502141286, + "maximum_absolute_error": 0.7265625 + }, + { + "row": 202, + "rms_error": 0.1433408438786218, + "maximum_absolute_error": 0.6875 + }, + { + "row": 154, + "rms_error": 0.13715977800436777, + "maximum_absolute_error": 0.8125 + }, + { + "row": 298, + "rms_error": 0.12126925277354424, + "maximum_absolute_error": 0.53125 + }, + { + "row": 399, + "rms_error": 0.1174680565810842, + "maximum_absolute_error": 0.40625 + }, + { + "row": 6, + "rms_error": 0.11535344107408924, + "maximum_absolute_error": 0.53125 + }, + { + "row": 305, + "rms_error": 0.11337162603771059, + "maximum_absolute_error": 0.59375 + }, + { + "row": 254, + "rms_error": 0.11291650606233904, + "maximum_absolute_error": 0.421875 + }, + { + "row": 34, + "rms_error": 0.10952884106608807, + "maximum_absolute_error": 0.4375 + }, + { + "row": 204, + "rms_error": 0.10401968151545389, + "maximum_absolute_error": 0.34375 + }, + { + "row": 375, + "rms_error": 0.10392484693056389, + "maximum_absolute_error": 0.359375 + }, + { + "row": 214, + "rms_error": 0.10219409674230935, + "maximum_absolute_error": 0.4375 + } + ], + "worst_columns": [ + { + "column": 2789, + "rms_error": 0.1578289865837652, + "maximum_absolute_error": 2.3125 + }, + { + "column": 17913, + "rms_error": 0.15538353652730102, + "maximum_absolute_error": 2.0625 + }, + { + "column": 42847, + "rms_error": 0.14889252980091122, + "maximum_absolute_error": 0.66796875 + }, + { + "column": 4538, + "rms_error": 0.14658752785401188, + "maximum_absolute_error": 1.5625 + }, + { + "column": 24629, + "rms_error": 0.14468194322460043, + "maximum_absolute_error": 1.9140625 + }, + { + "column": 89509, + "rms_error": 0.14284422361845295, + "maximum_absolute_error": 0.5859375 + }, + { + "column": 99838, + "rms_error": 0.14265954528891975, + "maximum_absolute_error": 1.3125 + }, + { + "column": 9473, + "rms_error": 0.14208920875775788, + "maximum_absolute_error": 0.78125 + }, + { + "column": 6405, + "rms_error": 0.1418762094509593, + "maximum_absolute_error": 0.625 + }, + { + "column": 11448, + "rms_error": 0.14118362756306777, + "maximum_absolute_error": 0.9921875 + }, + { + "column": 27370, + "rms_error": 0.1410649617828165, + "maximum_absolute_error": 0.634765625 + }, + { + "column": 26887, + "rms_error": 0.14071950646990417, + "maximum_absolute_error": 1.0 + }, + { + "column": 1286, + "rms_error": 0.14062844659738022, + "maximum_absolute_error": 1.6875 + }, + { + "column": 77351, + "rms_error": 0.1392467141340659, + "maximum_absolute_error": 1.84375 + }, + { + "column": 682, + "rms_error": 0.13845883762924746, + "maximum_absolute_error": 2.0625 + }, + { + "column": 3008, + "rms_error": 0.13844277725097628, + "maximum_absolute_error": 1.75 + }, + { + "column": 14014, + "rms_error": 0.13820224047033527, + "maximum_absolute_error": 0.546875 + }, + { + "column": 49132, + "rms_error": 0.13795923821016298, + "maximum_absolute_error": 0.583984375 + }, + { + "column": 45231, + "rms_error": 0.13789532696984333, + "maximum_absolute_error": 0.7412109375 + }, + { + "column": 8639, + "rms_error": 0.13764913746301133, + "maximum_absolute_error": 1.15625 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "1.7B-6-prose-513", + "phase": "prompt", + "native_logits_sha256": "b6c07fed3bdccf311a1a46b1b2dc3a5f3de069d0b0a433d679a25b41cfa15968", + "reference_logits_sha256": "3906de3fe32531a42940b5a11bb42bde5bfa5f7e574e65b1220e561aebe88f42", + "token_fixture_sha256": "4b9e533cc1948ee72c4231178ec3af21808d7542c97f35997bb9538161d9a4cb" + }, + { + "rows": 513, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.024179737262574948, + "maximum_absolute_error": 3.796875, + "differences": [], + "worst_rows": [ + { + "row": 489, + "rms_error": 0.5293546579338142, + "maximum_absolute_error": 2.7890625 + }, + { + "row": 366, + "rms_error": 0.4558905806185453, + "maximum_absolute_error": 3.796875 + }, + { + "row": 161, + "rms_error": 0.4345084173210362, + "maximum_absolute_error": 3.123046875 + }, + { + "row": 325, + "rms_error": 0.431749633352712, + "maximum_absolute_error": 2.630859375 + }, + { + "row": 120, + "rms_error": 0.39772496822398945, + "maximum_absolute_error": 2.265625 + }, + { + "row": 407, + "rms_error": 0.3518574975918526, + "maximum_absolute_error": 2.046875 + }, + { + "row": 243, + "rms_error": 0.21250835496186718, + "maximum_absolute_error": 1.25390625 + }, + { + "row": 41, + "rms_error": 0.17345533863740256, + "maximum_absolute_error": 0.59375 + }, + { + "row": 202, + "rms_error": 0.16639715715231237, + "maximum_absolute_error": 0.9296875 + }, + { + "row": 0, + "rms_error": 0.16594398697398952, + "maximum_absolute_error": 0.6640625 + }, + { + "row": 284, + "rms_error": 0.15998232374435067, + "maximum_absolute_error": 0.921875 + }, + { + "row": 448, + "rms_error": 0.1512733326946045, + "maximum_absolute_error": 0.8125 + }, + { + "row": 123, + "rms_error": 0.14537851888767206, + "maximum_absolute_error": 0.453125 + }, + { + "row": 432, + "rms_error": 0.12380107281166727, + "maximum_absolute_error": 0.453125 + }, + { + "row": 412, + "rms_error": 0.12124989134022418, + "maximum_absolute_error": 0.546875 + }, + { + "row": 164, + "rms_error": 0.11595322146785837, + "maximum_absolute_error": 0.40625 + }, + { + "row": 507, + "rms_error": 0.11504810528788113, + "maximum_absolute_error": 0.375 + }, + { + "row": 28, + "rms_error": 0.11302524768018539, + "maximum_absolute_error": 0.53125 + }, + { + "row": 205, + "rms_error": 0.10773378749727107, + "maximum_absolute_error": 0.390625 + }, + { + "row": 172, + "rms_error": 0.1032832534919513, + "maximum_absolute_error": 0.328125 + } + ], + "worst_columns": [ + { + "column": 10390, + "rms_error": 0.22725079140539975, + "maximum_absolute_error": 3.796875 + }, + { + "column": 57174, + "rms_error": 0.2209838995398379, + "maximum_absolute_error": 3.6796875 + }, + { + "column": 53654, + "rms_error": 0.21365533668939762, + "maximum_absolute_error": 3.53125 + }, + { + "column": 4528, + "rms_error": 0.21063222288494096, + "maximum_absolute_error": 3.4375 + }, + { + "column": 116814, + "rms_error": 0.19677703377873265, + "maximum_absolute_error": 2.7734375 + }, + { + "column": 2403, + "rms_error": 0.1920887890155629, + "maximum_absolute_error": 3.1826171875 + }, + { + "column": 89014, + "rms_error": 0.19111474562478636, + "maximum_absolute_error": 3.2421875 + }, + { + "column": 22938, + "rms_error": 0.18587160103955058, + "maximum_absolute_error": 3.0546875 + }, + { + "column": 55645, + "rms_error": 0.1853389358591622, + "maximum_absolute_error": 2.75 + }, + { + "column": 23062, + "rms_error": 0.18257677862175908, + "maximum_absolute_error": 2.53125 + }, + { + "column": 35285, + "rms_error": 0.17997155569634013, + "maximum_absolute_error": 3.03125 + }, + { + "column": 13238, + "rms_error": 0.1790858589252742, + "maximum_absolute_error": 2.8828125 + }, + { + "column": 5182, + "rms_error": 0.17426078157341826, + "maximum_absolute_error": 1.71875 + }, + { + "column": 6315, + "rms_error": 0.17036505675380553, + "maximum_absolute_error": 2.38671875 + }, + { + "column": 377, + "rms_error": 0.16983964120835898, + "maximum_absolute_error": 3.123046875 + }, + { + "column": 13162, + "rms_error": 0.16978807890366487, + "maximum_absolute_error": 2.09375 + }, + { + "column": 25737, + "rms_error": 0.16918303066883153, + "maximum_absolute_error": 3.03125 + }, + { + "column": 16934, + "rms_error": 0.16912821273237755, + "maximum_absolute_error": 2.3984375 + }, + { + "column": 1004, + "rms_error": 0.167766848855117, + "maximum_absolute_error": 2.36328125 + }, + { + "column": 586, + "rms_error": 0.16756588453265456, + "maximum_absolute_error": 2.625 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "1.7B-8-code-513", + "phase": "prompt", + "native_logits_sha256": "42aff32c97dbb25fabbd1415dbbb4b8f85b142f72fbc63b9a6666637a7047bb1", + "reference_logits_sha256": "0134bf8d533a7015ac1b07d452da9ab06266a83119d22d88f6c2d252969caa3d", + "token_fixture_sha256": "c932ae46fd83f6e834a4001bd27496af3ebf54ac841a694d03a64558ba0daacb" + }, + { + "rows": 513, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 1, + "relative_rms": 0.013054321718462684, + "maximum_absolute_error": 0.875, + "differences": [ + { + "row": 264, + "native_top": 207, + "reference_top": 2993, + "native_maximum_set": [ + 207 + ], + "reference_maximum_set": [ + 2993 + ], + "maximum_sets_overlap": false, + "native_margin": 0.25, + "reference_margin": 0.375, + "native_reference_contender_gap": 0.5, + "reference_native_contender_gap": 0.375 + } + ], + "worst_rows": [ + { + "row": 284, + "rms_error": 0.11660865974097531, + "maximum_absolute_error": 0.265625 + }, + { + "row": 38, + "rms_error": 0.11460334310170639, + "maximum_absolute_error": 0.328125 + }, + { + "row": 243, + "rms_error": 0.113975002379063, + "maximum_absolute_error": 0.4375 + }, + { + "row": 366, + "rms_error": 0.11019728140150578, + "maximum_absolute_error": 0.3203125 + }, + { + "row": 489, + "rms_error": 0.09779660019009252, + "maximum_absolute_error": 0.28125 + }, + { + "row": 161, + "rms_error": 0.09618629951635219, + "maximum_absolute_error": 0.328125 + }, + { + "row": 407, + "rms_error": 0.08630005056785829, + "maximum_absolute_error": 0.203125 + }, + { + "row": 106, + "rms_error": 0.08576597939678975, + "maximum_absolute_error": 0.21875 + }, + { + "row": 154, + "rms_error": 0.0846648037059979, + "maximum_absolute_error": 0.1875 + }, + { + "row": 426, + "rms_error": 0.08425171979022242, + "maximum_absolute_error": 0.328125 + }, + { + "row": 264, + "rms_error": 0.08412619776556655, + "maximum_absolute_error": 0.6875 + }, + { + "row": 29, + "rms_error": 0.07609650779142937, + "maximum_absolute_error": 0.248291015625 + }, + { + "row": 0, + "rms_error": 0.07317533579110798, + "maximum_absolute_error": 0.265625 + }, + { + "row": 437, + "rms_error": 0.07172593090335425, + "maximum_absolute_error": 0.21875 + }, + { + "row": 262, + "rms_error": 0.07145276927562015, + "maximum_absolute_error": 0.265625 + }, + { + "row": 126, + "rms_error": 0.06992320581155204, + "maximum_absolute_error": 0.2421875 + }, + { + "row": 88, + "rms_error": 0.06955772924510238, + "maximum_absolute_error": 0.25 + }, + { + "row": 448, + "rms_error": 0.06948356870027482, + "maximum_absolute_error": 0.875 + }, + { + "row": 287, + "rms_error": 0.06843449764216183, + "maximum_absolute_error": 0.234375 + }, + { + "row": 57, + "rms_error": 0.06819346713585736, + "maximum_absolute_error": 0.6875 + } + ], + "worst_columns": [ + { + "column": 20975, + "rms_error": 0.09383517087712488, + "maximum_absolute_error": 0.875 + }, + { + "column": 33237, + "rms_error": 0.08458998061074353, + "maximum_absolute_error": 0.375 + }, + { + "column": 5272, + "rms_error": 0.08397744240645438, + "maximum_absolute_error": 0.5 + }, + { + "column": 22497, + "rms_error": 0.08209502694226439, + "maximum_absolute_error": 0.7734375 + }, + { + "column": 16033, + "rms_error": 0.08206275163164192, + "maximum_absolute_error": 0.40625 + }, + { + "column": 8, + "rms_error": 0.079478434189202, + "maximum_absolute_error": 0.3125 + }, + { + "column": 15960, + "rms_error": 0.07668728183400887, + "maximum_absolute_error": 0.5078125 + }, + { + "column": 22657, + "rms_error": 0.07659736794176768, + "maximum_absolute_error": 0.5234375 + }, + { + "column": 62977, + "rms_error": 0.07499113542889467, + "maximum_absolute_error": 0.533203125 + }, + { + "column": 61089, + "rms_error": 0.07492330981344628, + "maximum_absolute_error": 0.375 + }, + { + "column": 116016, + "rms_error": 0.07387712012498947, + "maximum_absolute_error": 0.328125 + }, + { + "column": 18169, + "rms_error": 0.07252508158391092, + "maximum_absolute_error": 0.5537109375 + }, + { + "column": 42847, + "rms_error": 0.07160987062227049, + "maximum_absolute_error": 0.34375 + }, + { + "column": 22155, + "rms_error": 0.07056468983913752, + "maximum_absolute_error": 0.43359375 + }, + { + "column": 6, + "rms_error": 0.07002500717554196, + "maximum_absolute_error": 0.25 + }, + { + "column": 45707, + "rms_error": 0.06998371011317474, + "maximum_absolute_error": 0.328125 + }, + { + "column": 833, + "rms_error": 0.06996953885191347, + "maximum_absolute_error": 0.3125 + }, + { + "column": 49899, + "rms_error": 0.06959342544392765, + "maximum_absolute_error": 0.47265625 + }, + { + "column": 4484, + "rms_error": 0.06953165077986545, + "maximum_absolute_error": 0.421875 + }, + { + "column": 1224, + "rms_error": 0.06838360460343602, + "maximum_absolute_error": 0.375 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-4-code-513", + "phase": "prompt", + "native_logits_sha256": "83013e7646760ead9206c4a53b7cac56a19d11035629c876f9ea6701dddedc08", + "reference_logits_sha256": "e7780269883f67c353ee4da78c8838a83c1d3da53ae86749d2769ac9c5191d81", + "token_fixture_sha256": "c932ae46fd83f6e834a4001bd27496af3ebf54ac841a694d03a64558ba0daacb" + }, + { + "rows": 257, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 1, + "relative_rms": 0.008627400370203533, + "maximum_absolute_error": 0.53125, + "differences": [ + { + "row": 17, + "native_top": 296, + "reference_top": 405, + "native_maximum_set": [ + 296 + ], + "reference_maximum_set": [ + 405 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.125, + "native_reference_contender_gap": 0.1875, + "reference_native_contender_gap": 0.125 + }, + { + "row": 18, + "native_top": 22, + "reference_top": 1018, + "native_maximum_set": [ + 22, + 1018 + ], + "reference_maximum_set": [ + 1018 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + } + ], + "worst_rows": [ + { + "row": 243, + "rms_error": 0.09736034309484129, + "maximum_absolute_error": 0.25 + }, + { + "row": 13, + "rms_error": 0.08814603418682129, + "maximum_absolute_error": 0.3125 + }, + { + "row": 234, + "rms_error": 0.08733945420302694, + "maximum_absolute_error": 0.21875 + }, + { + "row": 150, + "rms_error": 0.07763974251918007, + "maximum_absolute_error": 0.2109375 + }, + { + "row": 75, + "rms_error": 0.07060574784723814, + "maximum_absolute_error": 0.40625 + }, + { + "row": 63, + "rms_error": 0.06972206208183651, + "maximum_absolute_error": 0.25 + }, + { + "row": 11, + "rms_error": 0.0685653823462518, + "maximum_absolute_error": 0.203125 + }, + { + "row": 30, + "rms_error": 0.06688009313489794, + "maximum_absolute_error": 0.21875 + }, + { + "row": 167, + "rms_error": 0.0656170326452077, + "maximum_absolute_error": 0.1875 + }, + { + "row": 228, + "rms_error": 0.06552723505195315, + "maximum_absolute_error": 0.171875 + }, + { + "row": 254, + "rms_error": 0.06412842885695617, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 176, + "rms_error": 0.06379156153384324, + "maximum_absolute_error": 0.25 + }, + { + "row": 165, + "rms_error": 0.06367008323722709, + "maximum_absolute_error": 0.1875 + }, + { + "row": 51, + "rms_error": 0.06359908072734448, + "maximum_absolute_error": 0.2265625 + }, + { + "row": 126, + "rms_error": 0.06253239432888728, + "maximum_absolute_error": 0.1875 + }, + { + "row": 62, + "rms_error": 0.06231537108257655, + "maximum_absolute_error": 0.19921875 + }, + { + "row": 142, + "rms_error": 0.06230262324636231, + "maximum_absolute_error": 0.21875 + }, + { + "row": 181, + "rms_error": 0.06150958820924385, + "maximum_absolute_error": 0.1875 + }, + { + "row": 216, + "rms_error": 0.06054421860258897, + "maximum_absolute_error": 0.25 + }, + { + "row": 195, + "rms_error": 0.0596384265326336, + "maximum_absolute_error": 0.171875 + } + ], + "worst_columns": [ + { + "column": 10625, + "rms_error": 0.08184762376752494, + "maximum_absolute_error": 0.40625 + }, + { + "column": 7161, + "rms_error": 0.07698907371429253, + "maximum_absolute_error": 0.5 + }, + { + "column": 10494, + "rms_error": 0.07586284643801129, + "maximum_absolute_error": 0.3125 + }, + { + "column": 334, + "rms_error": 0.07549601623584667, + "maximum_absolute_error": 0.3125 + }, + { + "column": 16420, + "rms_error": 0.07406697722604065, + "maximum_absolute_error": 0.375 + }, + { + "column": 2925, + "rms_error": 0.07397586540108855, + "maximum_absolute_error": 0.53125 + }, + { + "column": 3431, + "rms_error": 0.07335520334151921, + "maximum_absolute_error": 0.375 + }, + { + "column": 43345, + "rms_error": 0.0722360189328618, + "maximum_absolute_error": 0.38671875 + }, + { + "column": 677, + "rms_error": 0.07174925940196854, + "maximum_absolute_error": 0.25 + }, + { + "column": 229, + "rms_error": 0.07036525595960653, + "maximum_absolute_error": 0.25 + }, + { + "column": 50474, + "rms_error": 0.06988839649329859, + "maximum_absolute_error": 0.296875 + }, + { + "column": 20, + "rms_error": 0.06947663019402786, + "maximum_absolute_error": 0.3125 + }, + { + "column": 90208, + "rms_error": 0.06885264885154456, + "maximum_absolute_error": 0.40625 + }, + { + "column": 117761, + "rms_error": 0.06836584128965245, + "maximum_absolute_error": 0.34375 + }, + { + "column": 1840, + "rms_error": 0.0683339455991707, + "maximum_absolute_error": 0.28125 + }, + { + "column": 5435, + "rms_error": 0.06821270614006666, + "maximum_absolute_error": 0.28125 + }, + { + "column": 90167, + "rms_error": 0.06797470565489383, + "maximum_absolute_error": 0.3125 + }, + { + "column": 3464, + "rms_error": 0.06754028844248378, + "maximum_absolute_error": 0.34375 + }, + { + "column": 32564, + "rms_error": 0.06753482639543876, + "maximum_absolute_error": 0.40625 + }, + { + "column": 117113, + "rms_error": 0.06714711749105658, + "maximum_absolute_error": 0.328125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-4-reasoning-257", + "phase": "prompt", + "native_logits_sha256": "118440ccd7fc07a502485a3266a2878ec815f316bcfa9dddd859e5d324fb8654", + "reference_logits_sha256": "eaad83f4ec389f5c731c0d3422f2af47f218f8cdb220a6d0b752641c8f8f9bfe", + "token_fixture_sha256": "81449804fdd1a104d6dc9607c27d5cea49237064e50c80a765f68d8eba57f596" + }, + { + "rows": 513, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 1, + "relative_rms": 0.008214948624614303, + "maximum_absolute_error": 0.53125, + "differences": [ + { + "row": 17, + "native_top": 296, + "reference_top": 405, + "native_maximum_set": [ + 296 + ], + "reference_maximum_set": [ + 405 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.125, + "native_reference_contender_gap": 0.1875, + "reference_native_contender_gap": 0.125 + }, + { + "row": 18, + "native_top": 22, + "reference_top": 1018, + "native_maximum_set": [ + 22, + 1018 + ], + "reference_maximum_set": [ + 1018 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + } + ], + "worst_rows": [ + { + "row": 243, + "rms_error": 0.09736034309484129, + "maximum_absolute_error": 0.25 + }, + { + "row": 321, + "rms_error": 0.08859348692622644, + "maximum_absolute_error": 0.1875 + }, + { + "row": 13, + "rms_error": 0.08814603418682129, + "maximum_absolute_error": 0.3125 + }, + { + "row": 234, + "rms_error": 0.08733945420302694, + "maximum_absolute_error": 0.21875 + }, + { + "row": 150, + "rms_error": 0.07763974251918007, + "maximum_absolute_error": 0.2109375 + }, + { + "row": 281, + "rms_error": 0.07583519509884842, + "maximum_absolute_error": 0.1875 + }, + { + "row": 336, + "rms_error": 0.07530792503409797, + "maximum_absolute_error": 0.21875 + }, + { + "row": 75, + "rms_error": 0.07060574784723814, + "maximum_absolute_error": 0.40625 + }, + { + "row": 63, + "rms_error": 0.06972206208183651, + "maximum_absolute_error": 0.25 + }, + { + "row": 11, + "rms_error": 0.0685653823462518, + "maximum_absolute_error": 0.203125 + }, + { + "row": 294, + "rms_error": 0.06716212272945835, + "maximum_absolute_error": 0.4375 + }, + { + "row": 467, + "rms_error": 0.06709397685322804, + "maximum_absolute_error": 0.171875 + }, + { + "row": 30, + "rms_error": 0.06688009313489794, + "maximum_absolute_error": 0.21875 + }, + { + "row": 167, + "rms_error": 0.0656170326452077, + "maximum_absolute_error": 0.1875 + }, + { + "row": 228, + "rms_error": 0.06552723505195315, + "maximum_absolute_error": 0.171875 + }, + { + "row": 480, + "rms_error": 0.06471431251201011, + "maximum_absolute_error": 0.16796875 + }, + { + "row": 254, + "rms_error": 0.06412842885695617, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 176, + "rms_error": 0.06379156153384324, + "maximum_absolute_error": 0.25 + }, + { + "row": 165, + "rms_error": 0.06367008323722709, + "maximum_absolute_error": 0.1875 + }, + { + "row": 387, + "rms_error": 0.06359964520004042, + "maximum_absolute_error": 0.1875 + } + ], + "worst_columns": [ + { + "column": 677, + "rms_error": 0.07546077431715525, + "maximum_absolute_error": 0.34375 + }, + { + "column": 10625, + "rms_error": 0.07167506161265333, + "maximum_absolute_error": 0.40625 + }, + { + "column": 16420, + "rms_error": 0.06892223991389673, + "maximum_absolute_error": 0.375 + }, + { + "column": 334, + "rms_error": 0.06815249423517733, + "maximum_absolute_error": 0.3125 + }, + { + "column": 7161, + "rms_error": 0.06660870828360234, + "maximum_absolute_error": 0.5 + }, + { + "column": 10494, + "rms_error": 0.0660995629310206, + "maximum_absolute_error": 0.3125 + }, + { + "column": 1840, + "rms_error": 0.06598337756612141, + "maximum_absolute_error": 0.28125 + }, + { + "column": 2925, + "rms_error": 0.06581418699576513, + "maximum_absolute_error": 0.53125 + }, + { + "column": 229, + "rms_error": 0.06534062505054741, + "maximum_absolute_error": 0.25 + }, + { + "column": 90208, + "rms_error": 0.06468343639325762, + "maximum_absolute_error": 0.40625 + }, + { + "column": 3431, + "rms_error": 0.06452822259678301, + "maximum_absolute_error": 0.375 + }, + { + "column": 5435, + "rms_error": 0.0642164223197101, + "maximum_absolute_error": 0.28125 + }, + { + "column": 90167, + "rms_error": 0.06338073099560994, + "maximum_absolute_error": 0.3125 + }, + { + "column": 50474, + "rms_error": 0.06326483730155473, + "maximum_absolute_error": 0.296875 + }, + { + "column": 43345, + "rms_error": 0.06311631879887547, + "maximum_absolute_error": 0.38671875 + }, + { + "column": 20, + "rms_error": 0.06300834364964578, + "maximum_absolute_error": 0.3125 + }, + { + "column": 1196, + "rms_error": 0.06264076962447633, + "maximum_absolute_error": 0.25 + }, + { + "column": 21013, + "rms_error": 0.061497778582457266, + "maximum_absolute_error": 0.21875 + }, + { + "column": 46926, + "rms_error": 0.06091999014499288, + "maximum_absolute_error": 0.296875 + }, + { + "column": 15027, + "rms_error": 0.0606862275990072, + "maximum_absolute_error": 0.28125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-4-reasoning-513", + "phase": "prompt", + "native_logits_sha256": "7a2673d808c3fb08e430dccd995f02a0458d3b9a57342a985364f1542b73c13a", + "reference_logits_sha256": "52bd09f45fc04d6e30df13801df616755a4b6aeef0d3b29d30acafd911398c0f", + "token_fixture_sha256": "c8ce58a8d512375ea4635bc85d0ea8a235506aa56026087bda216ea61357a863" + }, + { + "rows": 257, + "vocabulary": 128000, + "top1_differences": 3, + "disjoint_maximum_sets": 1, + "relative_rms": 0.015280496666021706, + "maximum_absolute_error": 1.1875, + "differences": [ + { + "row": 13, + "native_top": 22, + "reference_top": 63908, + "native_maximum_set": [ + 22, + 63908 + ], + "reference_maximum_set": [ + 63908 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 22, + "native_top": 16033, + "reference_top": 91, + "native_maximum_set": [ + 16033 + ], + "reference_maximum_set": [ + 91 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.1875, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.1875 + }, + { + "row": 83, + "native_top": 3570, + "reference_top": 496, + "native_maximum_set": [ + 3570 + ], + "reference_maximum_set": [ + 496, + 3570 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.1875, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 57, + "rms_error": 0.16603965592691475, + "maximum_absolute_error": 0.96484375 + }, + { + "row": 98, + "rms_error": 0.13893384813257964, + "maximum_absolute_error": 1.1875 + }, + { + "row": 120, + "rms_error": 0.13592408554611574, + "maximum_absolute_error": 0.3291015625 + }, + { + "row": 247, + "rms_error": 0.10346569206693584, + "maximum_absolute_error": 0.53125 + }, + { + "row": 206, + "rms_error": 0.09054178951945603, + "maximum_absolute_error": 0.375 + }, + { + "row": 83, + "rms_error": 0.08761825280720467, + "maximum_absolute_error": 0.546875 + }, + { + "row": 55, + "rms_error": 0.08489186681149496, + "maximum_absolute_error": 0.3046875 + }, + { + "row": 243, + "rms_error": 0.08471422881700796, + "maximum_absolute_error": 0.58984375 + }, + { + "row": 13, + "rms_error": 0.07917376823524733, + "maximum_absolute_error": 0.28125 + }, + { + "row": 96, + "rms_error": 0.07844492352504183, + "maximum_absolute_error": 0.5625 + }, + { + "row": 111, + "rms_error": 0.07843442640209314, + "maximum_absolute_error": 0.203125 + }, + { + "row": 164, + "rms_error": 0.07456366576073772, + "maximum_absolute_error": 0.25 + }, + { + "row": 139, + "rms_error": 0.07346642214164588, + "maximum_absolute_error": 0.640625 + }, + { + "row": 222, + "rms_error": 0.07342471896768152, + "maximum_absolute_error": 0.203125 + }, + { + "row": 161, + "rms_error": 0.07268031627516287, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 182, + "rms_error": 0.0721782556203206, + "maximum_absolute_error": 0.3203125 + }, + { + "row": 218, + "rms_error": 0.07176656047450632, + "maximum_absolute_error": 0.234375 + }, + { + "row": 87, + "rms_error": 0.0705261637337868, + "maximum_absolute_error": 0.25 + }, + { + "row": 177, + "rms_error": 0.06972103875964812, + "maximum_absolute_error": 0.22265625 + }, + { + "row": 94, + "rms_error": 0.06668390158072966, + "maximum_absolute_error": 0.1953125 + } + ], + "worst_columns": [ + { + "column": 11460, + "rms_error": 0.10063212811146843, + "maximum_absolute_error": 0.96875 + }, + { + "column": 16033, + "rms_error": 0.09824492200653928, + "maximum_absolute_error": 0.625 + }, + { + "column": 33237, + "rms_error": 0.09768968714766435, + "maximum_absolute_error": 0.4375 + }, + { + "column": 723, + "rms_error": 0.09272147775661387, + "maximum_absolute_error": 0.5 + }, + { + "column": 12739, + "rms_error": 0.09102382675513906, + "maximum_absolute_error": 1.1875 + }, + { + "column": 43768, + "rms_error": 0.08821292990446143, + "maximum_absolute_error": 0.875 + }, + { + "column": 495, + "rms_error": 0.0881083745290935, + "maximum_absolute_error": 0.453125 + }, + { + "column": 42847, + "rms_error": 0.08768558572807102, + "maximum_absolute_error": 0.421875 + }, + { + "column": 27642, + "rms_error": 0.08718746414121055, + "maximum_absolute_error": 1.125 + }, + { + "column": 295, + "rms_error": 0.08699393124801501, + "maximum_absolute_error": 0.4375 + }, + { + "column": 6986, + "rms_error": 0.08609395212400547, + "maximum_absolute_error": 0.390625 + }, + { + "column": 93538, + "rms_error": 0.08596316593439618, + "maximum_absolute_error": 0.96484375 + }, + { + "column": 8, + "rms_error": 0.08541194490852676, + "maximum_absolute_error": 0.375 + }, + { + "column": 20975, + "rms_error": 0.08536186872382531, + "maximum_absolute_error": 0.515625 + }, + { + "column": 53560, + "rms_error": 0.0851040358119443, + "maximum_absolute_error": 0.90625 + }, + { + "column": 1046, + "rms_error": 0.08486800214496844, + "maximum_absolute_error": 0.5625 + }, + { + "column": 376, + "rms_error": 0.08452272249551593, + "maximum_absolute_error": 0.59375 + }, + { + "column": 5081, + "rms_error": 0.08434183062754937, + "maximum_absolute_error": 0.78125 + }, + { + "column": 893, + "rms_error": 0.08422992954434476, + "maximum_absolute_error": 0.53125 + }, + { + "column": 15960, + "rms_error": 0.08364418525466034, + "maximum_absolute_error": 0.75 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-code-257", + "phase": "prompt", + "native_logits_sha256": "93ea83c90c5623f4e17502a526e2358f9e1ba3f2d03999295745257dc95bbca6", + "reference_logits_sha256": "9b5568a07191137356488fa9410a007fe976ffa4fa07c61ab0f8705eea30e6d6", + "token_fixture_sha256": "96859365056d513a414e6eeef9d4a85bf72c3f9630d3e13a9ea0f57dd81b8bea" + }, + { + "rows": 513, + "vocabulary": 128000, + "top1_differences": 3, + "disjoint_maximum_sets": 1, + "relative_rms": 0.01387855605366047, + "maximum_absolute_error": 1.1875, + "differences": [ + { + "row": 13, + "native_top": 22, + "reference_top": 63908, + "native_maximum_set": [ + 22, + 63908 + ], + "reference_maximum_set": [ + 63908 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 22, + "native_top": 16033, + "reference_top": 91, + "native_maximum_set": [ + 16033 + ], + "reference_maximum_set": [ + 91 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.1875, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.1875 + }, + { + "row": 83, + "native_top": 3570, + "reference_top": 496, + "native_maximum_set": [ + 3570 + ], + "reference_maximum_set": [ + 496, + 3570 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.1875, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 57, + "rms_error": 0.16603965592691475, + "maximum_absolute_error": 0.96484375 + }, + { + "row": 98, + "rms_error": 0.13893384813257964, + "maximum_absolute_error": 1.1875 + }, + { + "row": 120, + "rms_error": 0.13592408554611574, + "maximum_absolute_error": 0.3291015625 + }, + { + "row": 407, + "rms_error": 0.10630926758662163, + "maximum_absolute_error": 0.96875 + }, + { + "row": 247, + "rms_error": 0.10346569206693584, + "maximum_absolute_error": 0.53125 + }, + { + "row": 206, + "rms_error": 0.09054178951945603, + "maximum_absolute_error": 0.375 + }, + { + "row": 83, + "rms_error": 0.08761825280720467, + "maximum_absolute_error": 0.546875 + }, + { + "row": 55, + "rms_error": 0.08489186681149496, + "maximum_absolute_error": 0.3046875 + }, + { + "row": 243, + "rms_error": 0.08471422881700796, + "maximum_absolute_error": 0.58984375 + }, + { + "row": 489, + "rms_error": 0.0837594941476296, + "maximum_absolute_error": 0.2265625 + }, + { + "row": 329, + "rms_error": 0.07984802170741474, + "maximum_absolute_error": 0.3125 + }, + { + "row": 13, + "rms_error": 0.07917376823524733, + "maximum_absolute_error": 0.28125 + }, + { + "row": 96, + "rms_error": 0.07844492352504183, + "maximum_absolute_error": 0.5625 + }, + { + "row": 111, + "rms_error": 0.07843442640209314, + "maximum_absolute_error": 0.203125 + }, + { + "row": 448, + "rms_error": 0.07548238121924124, + "maximum_absolute_error": 0.515625 + }, + { + "row": 164, + "rms_error": 0.07456366576073772, + "maximum_absolute_error": 0.25 + }, + { + "row": 139, + "rms_error": 0.07346642214164588, + "maximum_absolute_error": 0.640625 + }, + { + "row": 222, + "rms_error": 0.07342471896768152, + "maximum_absolute_error": 0.203125 + }, + { + "row": 433, + "rms_error": 0.07278386648661023, + "maximum_absolute_error": 0.15625 + }, + { + "row": 161, + "rms_error": 0.07268031627516287, + "maximum_absolute_error": 0.1953125 + } + ], + "worst_columns": [ + { + "column": 33237, + "rms_error": 0.09267241720449518, + "maximum_absolute_error": 0.4375 + }, + { + "column": 20975, + "rms_error": 0.09115123674004635, + "maximum_absolute_error": 0.96875 + }, + { + "column": 16033, + "rms_error": 0.08698415642060725, + "maximum_absolute_error": 0.625 + }, + { + "column": 22497, + "rms_error": 0.08229008437382075, + "maximum_absolute_error": 0.90625 + }, + { + "column": 495, + "rms_error": 0.08135343840816146, + "maximum_absolute_error": 0.515625 + }, + { + "column": 723, + "rms_error": 0.07985629015769032, + "maximum_absolute_error": 0.5 + }, + { + "column": 11460, + "rms_error": 0.07969094849400424, + "maximum_absolute_error": 0.96875 + }, + { + "column": 5272, + "rms_error": 0.07959740391069749, + "maximum_absolute_error": 0.671875 + }, + { + "column": 15960, + "rms_error": 0.0792037995287239, + "maximum_absolute_error": 0.75 + }, + { + "column": 295, + "rms_error": 0.07775311014731659, + "maximum_absolute_error": 0.4375 + }, + { + "column": 61089, + "rms_error": 0.07557352109763182, + "maximum_absolute_error": 0.34375 + }, + { + "column": 106633, + "rms_error": 0.07480329955192962, + "maximum_absolute_error": 0.40625 + }, + { + "column": 42847, + "rms_error": 0.07459951355028276, + "maximum_absolute_error": 0.421875 + }, + { + "column": 39382, + "rms_error": 0.07356328095121978, + "maximum_absolute_error": 0.328125 + }, + { + "column": 8, + "rms_error": 0.07281954490931111, + "maximum_absolute_error": 0.375 + }, + { + "column": 376, + "rms_error": 0.07256859509688507, + "maximum_absolute_error": 0.59375 + }, + { + "column": 1046, + "rms_error": 0.07223926169271816, + "maximum_absolute_error": 0.5625 + }, + { + "column": 1128, + "rms_error": 0.0721319674993418, + "maximum_absolute_error": 0.484375 + }, + { + "column": 30270, + "rms_error": 0.07166430767175694, + "maximum_absolute_error": 0.28125 + }, + { + "column": 116016, + "rms_error": 0.07153196281546222, + "maximum_absolute_error": 0.265625 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-code-513", + "phase": "prompt", + "native_logits_sha256": "3acb11b05fae7d193a867401fe358871782c9d63d2955eb77686cf09f0e97799", + "reference_logits_sha256": "252060b0a1076d72aa06510ea9b7decb5794d98e055588d39a7090f95c41c9b1", + "token_fixture_sha256": "c932ae46fd83f6e834a4001bd27496af3ebf54ac841a694d03a64558ba0daacb" + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 1, + "relative_rms": 0.015377440488712887, + "maximum_absolute_error": 0.96484375, + "differences": [ + { + "row": 13, + "native_top": 22, + "reference_top": 63908, + "native_maximum_set": [ + 22, + 63908 + ], + "reference_maximum_set": [ + 63908 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 22, + "native_top": 16033, + "reference_top": 91, + "native_maximum_set": [ + 16033 + ], + "reference_maximum_set": [ + 91 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.1875, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.1875 + } + ], + "worst_rows": [ + { + "row": 57, + "rms_error": 0.16603965592691475, + "maximum_absolute_error": 0.96484375 + }, + { + "row": 55, + "rms_error": 0.08489186681149496, + "maximum_absolute_error": 0.3046875 + }, + { + "row": 13, + "rms_error": 0.07917376823524733, + "maximum_absolute_error": 0.28125 + }, + { + "row": 2, + "rms_error": 0.06311253390330333, + "maximum_absolute_error": 0.216796875 + }, + { + "row": 32, + "rms_error": 0.060344014781392605, + "maximum_absolute_error": 0.28125 + }, + { + "row": 22, + "rms_error": 0.055970470902545984, + "maximum_absolute_error": 0.375 + }, + { + "row": 43, + "rms_error": 0.053908494189525466, + "maximum_absolute_error": 0.2109375 + }, + { + "row": 53, + "rms_error": 0.05371347456512, + "maximum_absolute_error": 0.2109375 + }, + { + "row": 50, + "rms_error": 0.05305531735124143, + "maximum_absolute_error": 0.1875 + }, + { + "row": 46, + "rms_error": 0.05286167453834549, + "maximum_absolute_error": 0.29296875 + }, + { + "row": 28, + "rms_error": 0.05198421343833805, + "maximum_absolute_error": 0.28125 + }, + { + "row": 56, + "rms_error": 0.050748718923888735, + "maximum_absolute_error": 0.203125 + }, + { + "row": 5, + "rms_error": 0.04965621029864758, + "maximum_absolute_error": 0.1669769287109375 + }, + { + "row": 34, + "rms_error": 0.04927833272243707, + "maximum_absolute_error": 0.21875 + }, + { + "row": 36, + "rms_error": 0.04908556341435462, + "maximum_absolute_error": 0.1875 + }, + { + "row": 4, + "rms_error": 0.0480743780157286, + "maximum_absolute_error": 0.294921875 + }, + { + "row": 62, + "rms_error": 0.04713935283875158, + "maximum_absolute_error": 0.3125 + }, + { + "row": 1, + "rms_error": 0.0456394677394624, + "maximum_absolute_error": 0.1875 + }, + { + "row": 33, + "rms_error": 0.0447680490608537, + "maximum_absolute_error": 0.21875 + }, + { + "row": 38, + "rms_error": 0.044176922297933914, + "maximum_absolute_error": 0.25 + } + ], + "worst_columns": [ + { + "column": 93538, + "rms_error": 0.14026637509168713, + "maximum_absolute_error": 0.96484375 + }, + { + "column": 538, + "rms_error": 0.13238456448014496, + "maximum_absolute_error": 0.953125 + }, + { + "column": 1046, + "rms_error": 0.1223944061953623, + "maximum_absolute_error": 0.5625 + }, + { + "column": 94852, + "rms_error": 0.1214250871538932, + "maximum_absolute_error": 0.828125 + }, + { + "column": 7272, + "rms_error": 0.11949435818918488, + "maximum_absolute_error": 0.84375 + }, + { + "column": 17, + "rms_error": 0.11716969672707923, + "maximum_absolute_error": 0.8125 + }, + { + "column": 6907, + "rms_error": 0.11613526003940478, + "maximum_absolute_error": 0.849609375 + }, + { + "column": 346, + "rms_error": 0.11392491089664815, + "maximum_absolute_error": 0.78125 + }, + { + "column": 16997, + "rms_error": 0.11332787179175711, + "maximum_absolute_error": 0.6875 + }, + { + "column": 3585, + "rms_error": 0.11256296995771509, + "maximum_absolute_error": 0.8125 + }, + { + "column": 647, + "rms_error": 0.11219731048893868, + "maximum_absolute_error": 0.796875 + }, + { + "column": 52421, + "rms_error": 0.10928653461278572, + "maximum_absolute_error": 0.796875 + }, + { + "column": 9673, + "rms_error": 0.10862716394009563, + "maximum_absolute_error": 0.7734375 + }, + { + "column": 13685, + "rms_error": 0.10791157272755565, + "maximum_absolute_error": 0.76953125 + }, + { + "column": 1954, + "rms_error": 0.10729993054205725, + "maximum_absolute_error": 0.767578125 + }, + { + "column": 1062, + "rms_error": 0.10728559453153616, + "maximum_absolute_error": 0.78125 + }, + { + "column": 7415, + "rms_error": 0.10701046101345699, + "maximum_absolute_error": 0.734375 + }, + { + "column": 5728, + "rms_error": 0.1068937588477719, + "maximum_absolute_error": 0.765625 + }, + { + "column": 48564, + "rms_error": 0.10625993545718224, + "maximum_absolute_error": 0.76806640625 + }, + { + "column": 34398, + "rms_error": 0.10505060026867931, + "maximum_absolute_error": 0.7001953125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-code-64", + "phase": "prompt", + "native_logits_sha256": "02225f1c8c7f5de05ebd44bd49948ad000aac040f3d6025d6655a9772ed08f9b", + "reference_logits_sha256": "d16f06c7813c231e4f3b7172456959da94d32adf024381666104fa28c0d3a859", + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748" + }, + { + "rows": 257, + "vocabulary": 128000, + "top1_differences": 4, + "disjoint_maximum_sets": 1, + "relative_rms": 0.008687223977156837, + "maximum_absolute_error": 0.4296875, + "differences": [ + { + "row": 17, + "native_top": 3547, + "reference_top": 3983, + "native_maximum_set": [ + 3547 + ], + "reference_maximum_set": [ + 3983 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 22, + "native_top": 22, + "reference_top": 20, + "native_maximum_set": [ + 22 + ], + "reference_maximum_set": [ + 20, + 22 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + }, + { + "row": 32, + "native_top": 3420, + "reference_top": 1030, + "native_maximum_set": [ + 3420 + ], + "reference_maximum_set": [ + 1030, + 3420 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + }, + { + "row": 38, + "native_top": 22, + "reference_top": 20, + "native_maximum_set": [ + 22 + ], + "reference_maximum_set": [ + 20, + 22 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 217, + "rms_error": 0.08379935117612562, + "maximum_absolute_error": 0.203125 + }, + { + "row": 23, + "rms_error": 0.08039388233086021, + "maximum_absolute_error": 0.3125 + }, + { + "row": 120, + "rms_error": 0.07907739150392885, + "maximum_absolute_error": 0.234375 + }, + { + "row": 7, + "rms_error": 0.07377735332353834, + "maximum_absolute_error": 0.2578125 + }, + { + "row": 255, + "rms_error": 0.07245671980606196, + "maximum_absolute_error": 0.15625 + }, + { + "row": 96, + "rms_error": 0.06554640927931134, + "maximum_absolute_error": 0.234375 + }, + { + "row": 181, + "rms_error": 0.06451813758698288, + "maximum_absolute_error": 0.16015625 + }, + { + "row": 46, + "rms_error": 0.06305097968923791, + "maximum_absolute_error": 0.25 + }, + { + "row": 32, + "rms_error": 0.058967015283877176, + "maximum_absolute_error": 0.21484375 + }, + { + "row": 215, + "rms_error": 0.05830500836073612, + "maximum_absolute_error": 0.1875 + }, + { + "row": 34, + "rms_error": 0.05736497199637097, + "maximum_absolute_error": 0.21875 + }, + { + "row": 152, + "rms_error": 0.0546517961829758, + "maximum_absolute_error": 0.15625 + }, + { + "row": 55, + "rms_error": 0.054592949647555664, + "maximum_absolute_error": 0.1875 + }, + { + "row": 111, + "rms_error": 0.05382973512156664, + "maximum_absolute_error": 0.1875 + }, + { + "row": 17, + "rms_error": 0.051758297630036205, + "maximum_absolute_error": 0.2783203125 + }, + { + "row": 6, + "rms_error": 0.05112465654439129, + "maximum_absolute_error": 0.25 + }, + { + "row": 13, + "rms_error": 0.05056898036695025, + "maximum_absolute_error": 0.1875 + }, + { + "row": 130, + "rms_error": 0.050396871568825684, + "maximum_absolute_error": 0.15625 + }, + { + "row": 129, + "rms_error": 0.04990677195990342, + "maximum_absolute_error": 0.15625 + }, + { + "row": 39, + "rms_error": 0.049882549125235924, + "maximum_absolute_error": 0.213623046875 + } + ], + "worst_columns": [ + { + "column": 13816, + "rms_error": 0.06947265953795, + "maximum_absolute_error": 0.3125 + }, + { + "column": 16526, + "rms_error": 0.06791369629868818, + "maximum_absolute_error": 0.265625 + }, + { + "column": 6986, + "rms_error": 0.0674144726640527, + "maximum_absolute_error": 0.25 + }, + { + "column": 49422, + "rms_error": 0.06615116485605516, + "maximum_absolute_error": 0.2734375 + }, + { + "column": 1030, + "rms_error": 0.0658893862431323, + "maximum_absolute_error": 0.3125 + }, + { + "column": 11448, + "rms_error": 0.0633983487576586, + "maximum_absolute_error": 0.3125 + }, + { + "column": 267, + "rms_error": 0.0630344443435407, + "maximum_absolute_error": 0.25 + }, + { + "column": 3420, + "rms_error": 0.06214520763861826, + "maximum_absolute_error": 0.1875 + }, + { + "column": 42847, + "rms_error": 0.06189670249833242, + "maximum_absolute_error": 0.234375 + }, + { + "column": 90978, + "rms_error": 0.06063410406891376, + "maximum_absolute_error": 0.25 + }, + { + "column": 20, + "rms_error": 0.06044664750164183, + "maximum_absolute_error": 0.1875 + }, + { + "column": 15763, + "rms_error": 0.0603795113950567, + "maximum_absolute_error": 0.25 + }, + { + "column": 29956, + "rms_error": 0.06037768491692642, + "maximum_absolute_error": 0.25 + }, + { + "column": 6542, + "rms_error": 0.05987515310587369, + "maximum_absolute_error": 0.25 + }, + { + "column": 30317, + "rms_error": 0.059700046743734185, + "maximum_absolute_error": 0.21875 + }, + { + "column": 309, + "rms_error": 0.05946428943100259, + "maximum_absolute_error": 0.1875 + }, + { + "column": 1712, + "rms_error": 0.05920268308837817, + "maximum_absolute_error": 0.29296875 + }, + { + "column": 26887, + "rms_error": 0.05905337347578534, + "maximum_absolute_error": 0.2890625 + }, + { + "column": 11622, + "rms_error": 0.058855048194446535, + "maximum_absolute_error": 0.4296875 + }, + { + "column": 9700, + "rms_error": 0.05876635844562529, + "maximum_absolute_error": 0.21875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-prose-257", + "phase": "prompt", + "native_logits_sha256": "9847c355bb21bff7af4f9a9db7953de32e794976757ac442379e0a9e25b97e1a", + "reference_logits_sha256": "d22537cee1af44f386d15510564af88e0cc0c09f57215e2c1109485b95945b94", + "token_fixture_sha256": "522ec0f7a7ef8481d2f5533e3962ddad1f418d4337b095282e40c096e2befb1d" + }, + { + "rows": 481, + "vocabulary": 128000, + "top1_differences": 4, + "disjoint_maximum_sets": 1, + "relative_rms": 0.008177575309897971, + "maximum_absolute_error": 0.4296875, + "differences": [ + { + "row": 17, + "native_top": 3547, + "reference_top": 3983, + "native_maximum_set": [ + 3547 + ], + "reference_maximum_set": [ + 3983 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 22, + "native_top": 22, + "reference_top": 20, + "native_maximum_set": [ + 22 + ], + "reference_maximum_set": [ + 20, + 22 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + }, + { + "row": 32, + "native_top": 3420, + "reference_top": 1030, + "native_maximum_set": [ + 3420 + ], + "reference_maximum_set": [ + 1030, + 3420 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + }, + { + "row": 38, + "native_top": 22, + "reference_top": 20, + "native_maximum_set": [ + 22 + ], + "reference_maximum_set": [ + 20, + 22 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 217, + "rms_error": 0.08379935117612562, + "maximum_absolute_error": 0.203125 + }, + { + "row": 23, + "rms_error": 0.08039388233086021, + "maximum_absolute_error": 0.3125 + }, + { + "row": 120, + "rms_error": 0.07907739150392885, + "maximum_absolute_error": 0.234375 + }, + { + "row": 7, + "rms_error": 0.07377735332353834, + "maximum_absolute_error": 0.2578125 + }, + { + "row": 255, + "rms_error": 0.07245671980606196, + "maximum_absolute_error": 0.15625 + }, + { + "row": 96, + "rms_error": 0.06554640927931134, + "maximum_absolute_error": 0.234375 + }, + { + "row": 181, + "rms_error": 0.06451813758698288, + "maximum_absolute_error": 0.16015625 + }, + { + "row": 46, + "rms_error": 0.06305097968923791, + "maximum_absolute_error": 0.25 + }, + { + "row": 263, + "rms_error": 0.06236134314363339, + "maximum_absolute_error": 0.15625 + }, + { + "row": 259, + "rms_error": 0.061297969511835367, + "maximum_absolute_error": 0.1875 + }, + { + "row": 379, + "rms_error": 0.06102616290689544, + "maximum_absolute_error": 0.1875 + }, + { + "row": 410, + "rms_error": 0.06000880202861097, + "maximum_absolute_error": 0.171875 + }, + { + "row": 32, + "rms_error": 0.058967015283877176, + "maximum_absolute_error": 0.21484375 + }, + { + "row": 215, + "rms_error": 0.05830500836073612, + "maximum_absolute_error": 0.1875 + }, + { + "row": 34, + "rms_error": 0.05736497199637097, + "maximum_absolute_error": 0.21875 + }, + { + "row": 284, + "rms_error": 0.05650241526171518, + "maximum_absolute_error": 0.1689453125 + }, + { + "row": 308, + "rms_error": 0.056441656147172746, + "maximum_absolute_error": 0.142578125 + }, + { + "row": 363, + "rms_error": 0.05628587663842697, + "maximum_absolute_error": 0.15625 + }, + { + "row": 364, + "rms_error": 0.054724031443787464, + "maximum_absolute_error": 0.15625 + }, + { + "row": 152, + "rms_error": 0.0546517961829758, + "maximum_absolute_error": 0.15625 + } + ], + "worst_columns": [ + { + "column": 16526, + "rms_error": 0.06479591924335233, + "maximum_absolute_error": 0.265625 + }, + { + "column": 6986, + "rms_error": 0.062563704354461, + "maximum_absolute_error": 0.25 + }, + { + "column": 13816, + "rms_error": 0.06201260012008843, + "maximum_absolute_error": 0.3125 + }, + { + "column": 49422, + "rms_error": 0.06110739642879905, + "maximum_absolute_error": 0.2734375 + }, + { + "column": 42847, + "rms_error": 0.05939664796486332, + "maximum_absolute_error": 0.25 + }, + { + "column": 1030, + "rms_error": 0.05831815180084154, + "maximum_absolute_error": 0.3125 + }, + { + "column": 3420, + "rms_error": 0.05801461119857934, + "maximum_absolute_error": 0.1875 + }, + { + "column": 20, + "rms_error": 0.056668062078217, + "maximum_absolute_error": 0.1875 + }, + { + "column": 11448, + "rms_error": 0.05644460167751568, + "maximum_absolute_error": 0.3125 + }, + { + "column": 267, + "rms_error": 0.05611997911451957, + "maximum_absolute_error": 0.25 + }, + { + "column": 90978, + "rms_error": 0.05547392851614682, + "maximum_absolute_error": 0.25 + }, + { + "column": 30317, + "rms_error": 0.05540269170052261, + "maximum_absolute_error": 0.21875 + }, + { + "column": 15763, + "rms_error": 0.05524383592158861, + "maximum_absolute_error": 0.25 + }, + { + "column": 29956, + "rms_error": 0.05503356515196459, + "maximum_absolute_error": 0.25 + }, + { + "column": 462, + "rms_error": 0.05491104413079826, + "maximum_absolute_error": 0.3125 + }, + { + "column": 9700, + "rms_error": 0.05485151319628807, + "maximum_absolute_error": 0.21875 + }, + { + "column": 309, + "rms_error": 0.054323453600986636, + "maximum_absolute_error": 0.1875 + }, + { + "column": 10393, + "rms_error": 0.05412014927595232, + "maximum_absolute_error": 0.2109375 + }, + { + "column": 89509, + "rms_error": 0.054021113051263996, + "maximum_absolute_error": 0.21875 + }, + { + "column": 22, + "rms_error": 0.05361337721648677, + "maximum_absolute_error": 0.1875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-prose-513", + "phase": "prompt", + "native_logits_sha256": "4633f10a1ef2861c9e8a6b6e2af3b001516d8f7dfacda7c74afb24ce0771a007", + "reference_logits_sha256": "bbb6ab06a5d755e71e77c15a17908ed3e31876435bd903d049990c00d0b8830c", + "token_fixture_sha256": "4b9e533cc1948ee72c4231178ec3af21808d7542c97f35997bb9538161d9a4cb" + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 4, + "disjoint_maximum_sets": 1, + "relative_rms": 0.010571638126272737, + "maximum_absolute_error": 0.4296875, + "differences": [ + { + "row": 17, + "native_top": 3547, + "reference_top": 3983, + "native_maximum_set": [ + 3547 + ], + "reference_maximum_set": [ + 3983 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 22, + "native_top": 22, + "reference_top": 20, + "native_maximum_set": [ + 22 + ], + "reference_maximum_set": [ + 20, + 22 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + }, + { + "row": 32, + "native_top": 3420, + "reference_top": 1030, + "native_maximum_set": [ + 3420 + ], + "reference_maximum_set": [ + 1030, + 3420 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + }, + { + "row": 38, + "native_top": 22, + "reference_top": 20, + "native_maximum_set": [ + 22 + ], + "reference_maximum_set": [ + 20, + 22 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 23, + "rms_error": 0.08039388233086021, + "maximum_absolute_error": 0.3125 + }, + { + "row": 7, + "rms_error": 0.07377735332353834, + "maximum_absolute_error": 0.2578125 + }, + { + "row": 46, + "rms_error": 0.06305097968923791, + "maximum_absolute_error": 0.25 + }, + { + "row": 32, + "rms_error": 0.058967015283877176, + "maximum_absolute_error": 0.21484375 + }, + { + "row": 34, + "rms_error": 0.05736497199637097, + "maximum_absolute_error": 0.21875 + }, + { + "row": 55, + "rms_error": 0.054592949647555664, + "maximum_absolute_error": 0.1875 + }, + { + "row": 17, + "rms_error": 0.051758297630036205, + "maximum_absolute_error": 0.2783203125 + }, + { + "row": 6, + "rms_error": 0.05112465654439129, + "maximum_absolute_error": 0.25 + }, + { + "row": 13, + "rms_error": 0.05056898036695025, + "maximum_absolute_error": 0.1875 + }, + { + "row": 39, + "rms_error": 0.049882549125235924, + "maximum_absolute_error": 0.213623046875 + }, + { + "row": 61, + "rms_error": 0.04838702377379141, + "maximum_absolute_error": 0.15625 + }, + { + "row": 58, + "rms_error": 0.047649187244348426, + "maximum_absolute_error": 0.2421875 + }, + { + "row": 33, + "rms_error": 0.047454539265684145, + "maximum_absolute_error": 0.3125 + }, + { + "row": 56, + "rms_error": 0.047372303043136416, + "maximum_absolute_error": 0.171875 + }, + { + "row": 2, + "rms_error": 0.047287506612965116, + "maximum_absolute_error": 0.296875 + }, + { + "row": 25, + "rms_error": 0.04604573401659605, + "maximum_absolute_error": 0.234375 + }, + { + "row": 41, + "rms_error": 0.044305615353516105, + "maximum_absolute_error": 0.21875 + }, + { + "row": 50, + "rms_error": 0.04426982628946516, + "maximum_absolute_error": 0.15625 + }, + { + "row": 0, + "rms_error": 0.04364004660563574, + "maximum_absolute_error": 0.29345703125 + }, + { + "row": 21, + "rms_error": 0.043581891994395584, + "maximum_absolute_error": 0.251953125 + } + ], + "worst_columns": [ + { + "column": 11622, + "rms_error": 0.08676552033493201, + "maximum_absolute_error": 0.4296875 + }, + { + "column": 13816, + "rms_error": 0.08661855364099594, + "maximum_absolute_error": 0.3125 + }, + { + "column": 6542, + "rms_error": 0.08416064195189571, + "maximum_absolute_error": 0.25 + }, + { + "column": 1030, + "rms_error": 0.08382524896044896, + "maximum_absolute_error": 0.3125 + }, + { + "column": 8818, + "rms_error": 0.08311185627337121, + "maximum_absolute_error": 0.21875 + }, + { + "column": 6986, + "rms_error": 0.08241246969788624, + "maximum_absolute_error": 0.21875 + }, + { + "column": 11448, + "rms_error": 0.08143089496149553, + "maximum_absolute_error": 0.3125 + }, + { + "column": 267, + "rms_error": 0.08106204079973661, + "maximum_absolute_error": 0.25 + }, + { + "column": 3420, + "rms_error": 0.07859030085592413, + "maximum_absolute_error": 0.1875 + }, + { + "column": 1712, + "rms_error": 0.07718456039798838, + "maximum_absolute_error": 0.29296875 + }, + { + "column": 35394, + "rms_error": 0.076781310787107, + "maximum_absolute_error": 0.25 + }, + { + "column": 11627, + "rms_error": 0.07658840271631162, + "maximum_absolute_error": 0.279052734375 + }, + { + "column": 3599, + "rms_error": 0.0760596010318637, + "maximum_absolute_error": 0.21875 + }, + { + "column": 18383, + "rms_error": 0.07604633596249258, + "maximum_absolute_error": 0.3125 + }, + { + "column": 19235, + "rms_error": 0.07573209869492893, + "maximum_absolute_error": 0.218994140625 + }, + { + "column": 462, + "rms_error": 0.07469266865614431, + "maximum_absolute_error": 0.25 + }, + { + "column": 42847, + "rms_error": 0.07458991176325436, + "maximum_absolute_error": 0.1875 + }, + { + "column": 9263, + "rms_error": 0.0745754146085657, + "maximum_absolute_error": 0.251953125 + }, + { + "column": 5877, + "rms_error": 0.0743898913314104, + "maximum_absolute_error": 0.19140625 + }, + { + "column": 97278, + "rms_error": 0.07425267650337575, + "maximum_absolute_error": 0.1640625 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-prose-64", + "phase": "prompt", + "native_logits_sha256": "e494f59f8738cb12a4329b1ba0c034861541e2d11fc8b31401b9ed212f3c67ae", + "reference_logits_sha256": "ac19ea6513313dae9ed3b5a46560ec89cefeaf2cfc97fd7dd8b9073b978868ac", + "token_fixture_sha256": "9c3636ee22fb5d1a9d4d6e3cc0b8e47cce20cb4288bb16f2970de8d55ee27708" + }, + { + "rows": 257, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 1, + "relative_rms": 0.009898396969534574, + "maximum_absolute_error": 0.5, + "differences": [ + { + "row": 5, + "native_top": 334, + "reference_top": 440, + "native_maximum_set": [ + 334, + 440 + ], + "reference_maximum_set": [ + 440 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 24, + "native_top": 27, + "reference_top": 1325, + "native_maximum_set": [ + 27 + ], + "reference_maximum_set": [ + 1325 + ], + "maximum_sets_overlap": false, + "native_margin": 0.09375, + "reference_margin": 0.15625, + "native_reference_contender_gap": 0.09375, + "reference_native_contender_gap": 0.15625 + } + ], + "worst_rows": [ + { + "row": 9, + "rms_error": 0.12294932383509793, + "maximum_absolute_error": 0.3125 + }, + { + "row": 103, + "rms_error": 0.10160416220105774, + "maximum_absolute_error": 0.28125 + }, + { + "row": 110, + "rms_error": 0.10114354913995284, + "maximum_absolute_error": 0.5 + }, + { + "row": 7, + "rms_error": 0.09792788416349865, + "maximum_absolute_error": 0.25 + }, + { + "row": 165, + "rms_error": 0.09590905155847587, + "maximum_absolute_error": 0.25 + }, + { + "row": 21, + "rms_error": 0.09219132467004232, + "maximum_absolute_error": 0.359375 + }, + { + "row": 141, + "rms_error": 0.08782615561660322, + "maximum_absolute_error": 0.25 + }, + { + "row": 58, + "rms_error": 0.08683439196527183, + "maximum_absolute_error": 0.4375 + }, + { + "row": 85, + "rms_error": 0.07975448421498577, + "maximum_absolute_error": 0.234375 + }, + { + "row": 89, + "rms_error": 0.0781388753515518, + "maximum_absolute_error": 0.265625 + }, + { + "row": 13, + "rms_error": 0.076647021407893, + "maximum_absolute_error": 0.25 + }, + { + "row": 19, + "rms_error": 0.07556712337015668, + "maximum_absolute_error": 0.203125 + }, + { + "row": 114, + "rms_error": 0.07536100251583458, + "maximum_absolute_error": 0.25 + }, + { + "row": 5, + "rms_error": 0.0747696754205199, + "maximum_absolute_error": 0.2421875 + }, + { + "row": 30, + "rms_error": 0.07406965268320222, + "maximum_absolute_error": 0.375 + }, + { + "row": 102, + "rms_error": 0.07096236477801798, + "maximum_absolute_error": 0.4375 + }, + { + "row": 107, + "rms_error": 0.06980058541020907, + "maximum_absolute_error": 0.21875 + }, + { + "row": 42, + "rms_error": 0.0696342552253152, + "maximum_absolute_error": 0.21875 + }, + { + "row": 200, + "rms_error": 0.06743500902392179, + "maximum_absolute_error": 0.203125 + }, + { + "row": 44, + "rms_error": 0.06676616207605728, + "maximum_absolute_error": 0.234375 + } + ], + "worst_columns": [ + { + "column": 1703, + "rms_error": 0.0856645132484912, + "maximum_absolute_error": 0.40625 + }, + { + "column": 117761, + "rms_error": 0.08207695043348325, + "maximum_absolute_error": 0.296875 + }, + { + "column": 1840, + "rms_error": 0.08204436219477908, + "maximum_absolute_error": 0.4375 + }, + { + "column": 405, + "rms_error": 0.07985900923179519, + "maximum_absolute_error": 0.375 + }, + { + "column": 10494, + "rms_error": 0.07946168344183437, + "maximum_absolute_error": 0.3125 + }, + { + "column": 229, + "rms_error": 0.07926958622905536, + "maximum_absolute_error": 0.3125 + }, + { + "column": 4475, + "rms_error": 0.0781806409979514, + "maximum_absolute_error": 0.2578125 + }, + { + "column": 334, + "rms_error": 0.07754743095485074, + "maximum_absolute_error": 0.359375 + }, + { + "column": 2925, + "rms_error": 0.07676855092933686, + "maximum_absolute_error": 0.375 + }, + { + "column": 25543, + "rms_error": 0.07543490089297242, + "maximum_absolute_error": 0.25 + }, + { + "column": 1008, + "rms_error": 0.07478947115804738, + "maximum_absolute_error": 0.3125 + }, + { + "column": 10625, + "rms_error": 0.07356928406888709, + "maximum_absolute_error": 0.25 + }, + { + "column": 1394, + "rms_error": 0.07236199487770618, + "maximum_absolute_error": 0.3125 + }, + { + "column": 16420, + "rms_error": 0.07227397651406046, + "maximum_absolute_error": 0.375 + }, + { + "column": 20, + "rms_error": 0.07203799413678999, + "maximum_absolute_error": 0.25 + }, + { + "column": 3464, + "rms_error": 0.07145507540377057, + "maximum_absolute_error": 0.3125 + }, + { + "column": 2735, + "rms_error": 0.07130990354831157, + "maximum_absolute_error": 0.240234375 + }, + { + "column": 12515, + "rms_error": 0.07071132995069165, + "maximum_absolute_error": 0.34033203125 + }, + { + "column": 496, + "rms_error": 0.07057041703818662, + "maximum_absolute_error": 0.5 + }, + { + "column": 93007, + "rms_error": 0.07040472957602888, + "maximum_absolute_error": 0.2265625 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-reasoning-257", + "phase": "prompt", + "native_logits_sha256": "9be9b95af0ff0c00697f8737ce9271ccac0e6dcd519b56d49f3666623ae6fd2c", + "reference_logits_sha256": "13287ae65d9ab1d56d292182bb95d953d2fae7cef5b72c7363c76bb7a64bb0a7", + "token_fixture_sha256": "81449804fdd1a104d6dc9607c27d5cea49237064e50c80a765f68d8eba57f596" + }, + { + "rows": 513, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 1, + "relative_rms": 0.010044146878069994, + "maximum_absolute_error": 0.5, + "differences": [ + { + "row": 5, + "native_top": 334, + "reference_top": 440, + "native_maximum_set": [ + 334, + 440 + ], + "reference_maximum_set": [ + 440 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 24, + "native_top": 27, + "reference_top": 1325, + "native_maximum_set": [ + 27 + ], + "reference_maximum_set": [ + 1325 + ], + "maximum_sets_overlap": false, + "native_margin": 0.09375, + "reference_margin": 0.15625, + "native_reference_contender_gap": 0.09375, + "reference_native_contender_gap": 0.15625 + } + ], + "worst_rows": [ + { + "row": 336, + "rms_error": 0.16609206457171105, + "maximum_absolute_error": 0.3125 + }, + { + "row": 356, + "rms_error": 0.14023515464406264, + "maximum_absolute_error": 0.25 + }, + { + "row": 453, + "rms_error": 0.1372254272694266, + "maximum_absolute_error": 0.28125 + }, + { + "row": 9, + "rms_error": 0.12294932383509793, + "maximum_absolute_error": 0.3125 + }, + { + "row": 258, + "rms_error": 0.12051410885812569, + "maximum_absolute_error": 0.3125 + }, + { + "row": 342, + "rms_error": 0.11386100024894089, + "maximum_absolute_error": 0.25 + }, + { + "row": 103, + "rms_error": 0.10160416220105774, + "maximum_absolute_error": 0.28125 + }, + { + "row": 110, + "rms_error": 0.10114354913995284, + "maximum_absolute_error": 0.5 + }, + { + "row": 7, + "rms_error": 0.09792788416349865, + "maximum_absolute_error": 0.25 + }, + { + "row": 297, + "rms_error": 0.09752375271030087, + "maximum_absolute_error": 0.25 + }, + { + "row": 165, + "rms_error": 0.09590905155847587, + "maximum_absolute_error": 0.25 + }, + { + "row": 21, + "rms_error": 0.09219132467004232, + "maximum_absolute_error": 0.359375 + }, + { + "row": 141, + "rms_error": 0.08782615561660322, + "maximum_absolute_error": 0.25 + }, + { + "row": 58, + "rms_error": 0.08683439196527183, + "maximum_absolute_error": 0.4375 + }, + { + "row": 262, + "rms_error": 0.08668250033316892, + "maximum_absolute_error": 0.265625 + }, + { + "row": 316, + "rms_error": 0.08395940976126434, + "maximum_absolute_error": 0.1875 + }, + { + "row": 285, + "rms_error": 0.08306386862012309, + "maximum_absolute_error": 0.21875 + }, + { + "row": 480, + "rms_error": 0.08126051445884339, + "maximum_absolute_error": 0.21875 + }, + { + "row": 398, + "rms_error": 0.07998733143285677, + "maximum_absolute_error": 0.22265625 + }, + { + "row": 85, + "rms_error": 0.07975448421498577, + "maximum_absolute_error": 0.234375 + } + ], + "worst_columns": [ + { + "column": 1394, + "rms_error": 0.07723993823190402, + "maximum_absolute_error": 0.3125 + }, + { + "column": 334, + "rms_error": 0.0762598215750573, + "maximum_absolute_error": 0.359375 + }, + { + "column": 229, + "rms_error": 0.07471747871425395, + "maximum_absolute_error": 0.3125 + }, + { + "column": 10494, + "rms_error": 0.07315162418573169, + "maximum_absolute_error": 0.3125 + }, + { + "column": 1703, + "rms_error": 0.0731266475167824, + "maximum_absolute_error": 0.40625 + }, + { + "column": 405, + "rms_error": 0.07294057710584206, + "maximum_absolute_error": 0.375 + }, + { + "column": 1840, + "rms_error": 0.07268557762292455, + "maximum_absolute_error": 0.4375 + }, + { + "column": 117761, + "rms_error": 0.07215104785352623, + "maximum_absolute_error": 0.296875 + }, + { + "column": 1008, + "rms_error": 0.07148034781113778, + "maximum_absolute_error": 0.375 + }, + { + "column": 4475, + "rms_error": 0.07049110050672319, + "maximum_absolute_error": 0.2578125 + }, + { + "column": 10625, + "rms_error": 0.0698603888152247, + "maximum_absolute_error": 0.25 + }, + { + "column": 3464, + "rms_error": 0.06890850250774827, + "maximum_absolute_error": 0.3125 + }, + { + "column": 20, + "rms_error": 0.06866674579088854, + "maximum_absolute_error": 0.25 + }, + { + "column": 16420, + "rms_error": 0.06837060342692489, + "maximum_absolute_error": 0.375 + }, + { + "column": 50523, + "rms_error": 0.06747849983876673, + "maximum_absolute_error": 0.265625 + }, + { + "column": 3300, + "rms_error": 0.06746695747737004, + "maximum_absolute_error": 0.265625 + }, + { + "column": 2925, + "rms_error": 0.067436494751403, + "maximum_absolute_error": 0.375 + }, + { + "column": 7575, + "rms_error": 0.06727481676363273, + "maximum_absolute_error": 0.234375 + }, + { + "column": 677, + "rms_error": 0.06672910517128115, + "maximum_absolute_error": 0.28125 + }, + { + "column": 3431, + "rms_error": 0.06654209504854995, + "maximum_absolute_error": 0.296875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-reasoning-513", + "phase": "prompt", + "native_logits_sha256": "049ebf779f8b00545a73253fb30fcd55cb1343321cb2f035fd5ad725153ddb66", + "reference_logits_sha256": "c2b614889684260162edc83cccba916488b87f1fcca785d5640fe037a195127b", + "token_fixture_sha256": "c8ce58a8d512375ea4635bc85d0ea8a235506aa56026087bda216ea61357a863" + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 1, + "relative_rms": 0.010343178217235415, + "maximum_absolute_error": 0.484375, + "differences": [ + { + "row": 5, + "native_top": 334, + "reference_top": 440, + "native_maximum_set": [ + 334, + 440 + ], + "reference_maximum_set": [ + 440 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 24, + "native_top": 27, + "reference_top": 1325, + "native_maximum_set": [ + 27 + ], + "reference_maximum_set": [ + 1325 + ], + "maximum_sets_overlap": false, + "native_margin": 0.09375, + "reference_margin": 0.15625, + "native_reference_contender_gap": 0.09375, + "reference_native_contender_gap": 0.15625 + } + ], + "worst_rows": [ + { + "row": 9, + "rms_error": 0.12294932383509793, + "maximum_absolute_error": 0.3125 + }, + { + "row": 7, + "rms_error": 0.09792788416349865, + "maximum_absolute_error": 0.25 + }, + { + "row": 21, + "rms_error": 0.09219132467004232, + "maximum_absolute_error": 0.359375 + }, + { + "row": 58, + "rms_error": 0.08683439196527183, + "maximum_absolute_error": 0.4375 + }, + { + "row": 13, + "rms_error": 0.076647021407893, + "maximum_absolute_error": 0.25 + }, + { + "row": 19, + "rms_error": 0.07556712337015668, + "maximum_absolute_error": 0.203125 + }, + { + "row": 5, + "rms_error": 0.0747696754205199, + "maximum_absolute_error": 0.2421875 + }, + { + "row": 30, + "rms_error": 0.07406965268320222, + "maximum_absolute_error": 0.375 + }, + { + "row": 42, + "rms_error": 0.0696342552253152, + "maximum_absolute_error": 0.21875 + }, + { + "row": 44, + "rms_error": 0.06676616207605728, + "maximum_absolute_error": 0.234375 + }, + { + "row": 57, + "rms_error": 0.0651700931470314, + "maximum_absolute_error": 0.2197265625 + }, + { + "row": 50, + "rms_error": 0.06460902101158113, + "maximum_absolute_error": 0.28125 + }, + { + "row": 60, + "rms_error": 0.06213349006348324, + "maximum_absolute_error": 0.484375 + }, + { + "row": 37, + "rms_error": 0.05803985632460466, + "maximum_absolute_error": 0.2265625 + }, + { + "row": 24, + "rms_error": 0.05769156682192908, + "maximum_absolute_error": 0.1875 + }, + { + "row": 35, + "rms_error": 0.05589824270837972, + "maximum_absolute_error": 0.1875 + }, + { + "row": 51, + "rms_error": 0.055274640833256124, + "maximum_absolute_error": 0.3388671875 + }, + { + "row": 63, + "rms_error": 0.05517972084767051, + "maximum_absolute_error": 0.25 + }, + { + "row": 4, + "rms_error": 0.05461585530859304, + "maximum_absolute_error": 0.171875 + }, + { + "row": 48, + "rms_error": 0.05397405183852856, + "maximum_absolute_error": 0.25 + } + ], + "worst_columns": [ + { + "column": 1840, + "rms_error": 0.10111460592939618, + "maximum_absolute_error": 0.4375 + }, + { + "column": 10494, + "rms_error": 0.09841963893580742, + "maximum_absolute_error": 0.28125 + }, + { + "column": 1703, + "rms_error": 0.09518479093026615, + "maximum_absolute_error": 0.25 + }, + { + "column": 4475, + "rms_error": 0.09324190418598297, + "maximum_absolute_error": 0.25 + }, + { + "column": 229, + "rms_error": 0.09315076689145957, + "maximum_absolute_error": 0.25 + }, + { + "column": 2258, + "rms_error": 0.09059171723173802, + "maximum_absolute_error": 0.484375 + }, + { + "column": 334, + "rms_error": 0.09045782995326458, + "maximum_absolute_error": 0.359375 + }, + { + "column": 61851, + "rms_error": 0.09000283130854655, + "maximum_absolute_error": 0.25 + }, + { + "column": 90352, + "rms_error": 0.08870560839179299, + "maximum_absolute_error": 0.3125 + }, + { + "column": 405, + "rms_error": 0.08719888373544142, + "maximum_absolute_error": 0.375 + }, + { + "column": 20, + "rms_error": 0.08526906231618388, + "maximum_absolute_error": 0.1875 + }, + { + "column": 677, + "rms_error": 0.08518513880680806, + "maximum_absolute_error": 0.25 + }, + { + "column": 25543, + "rms_error": 0.0835634989152927, + "maximum_absolute_error": 0.1875 + }, + { + "column": 117761, + "rms_error": 0.0832777110193787, + "maximum_absolute_error": 0.28125 + }, + { + "column": 34828, + "rms_error": 0.08286824694843019, + "maximum_absolute_error": 0.203125 + }, + { + "column": 11966, + "rms_error": 0.08267441158840844, + "maximum_absolute_error": 0.3125 + }, + { + "column": 62274, + "rms_error": 0.08258307002770117, + "maximum_absolute_error": 0.203125 + }, + { + "column": 93524, + "rms_error": 0.08253226097380381, + "maximum_absolute_error": 0.234375 + }, + { + "column": 12515, + "rms_error": 0.08247151764368085, + "maximum_absolute_error": 0.21484375 + }, + { + "column": 27, + "rms_error": 0.08232225812006795, + "maximum_absolute_error": 0.23175048828125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-reasoning-64", + "phase": "prompt", + "native_logits_sha256": "128b568000f7ac49b194d340bfea43df9950db1dbadc3da8983b2100252da128", + "reference_logits_sha256": "f681dc6a66bf91858634b90aa1ee0c4c541e4775675b5042412c93bb4fd6ccfa", + "token_fixture_sha256": "f3b50ade94353156b86075d186c03eae5a91e007593c314c55e19575b9c8f8eb" + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "relative_rms": 0.011164950262903948, + "maximum_absolute_error": 0.5, + "differences": [ + { + "row": 0, + "native_top": 278, + "reference_top": 20, + "native_maximum_set": [ + 278 + ], + "reference_maximum_set": [ + 20, + 278 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + }, + { + "row": 10, + "native_top": 10513, + "reference_top": 4986, + "native_maximum_set": [ + 10513 + ], + "reference_maximum_set": [ + 4986, + 10513 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 10, + "rms_error": 0.09551921061476651, + "maximum_absolute_error": 0.375 + }, + { + "row": 39, + "rms_error": 0.08395210342081126, + "maximum_absolute_error": 0.25 + }, + { + "row": 25, + "rms_error": 0.08320553966428762, + "maximum_absolute_error": 0.27734375 + }, + { + "row": 46, + "rms_error": 0.0706573408786518, + "maximum_absolute_error": 0.25 + }, + { + "row": 23, + "rms_error": 0.06548567329110504, + "maximum_absolute_error": 0.25 + }, + { + "row": 4, + "rms_error": 0.06060751068936593, + "maximum_absolute_error": 0.21875 + }, + { + "row": 33, + "rms_error": 0.05619865015537761, + "maximum_absolute_error": 0.5 + }, + { + "row": 36, + "rms_error": 0.05212875239855823, + "maximum_absolute_error": 0.1875 + }, + { + "row": 28, + "rms_error": 0.051282193924818816, + "maximum_absolute_error": 0.15625 + }, + { + "row": 62, + "rms_error": 0.05001288854501931, + "maximum_absolute_error": 0.25 + }, + { + "row": 50, + "rms_error": 0.04987065215628033, + "maximum_absolute_error": 0.21875 + }, + { + "row": 17, + "rms_error": 0.04820842257785237, + "maximum_absolute_error": 0.1875 + }, + { + "row": 45, + "rms_error": 0.04630302236725418, + "maximum_absolute_error": 0.203125 + }, + { + "row": 60, + "rms_error": 0.046181097928298824, + "maximum_absolute_error": 0.1640625 + }, + { + "row": 49, + "rms_error": 0.046115042054958866, + "maximum_absolute_error": 0.21875 + }, + { + "row": 48, + "rms_error": 0.04557910008148842, + "maximum_absolute_error": 0.25 + }, + { + "row": 59, + "rms_error": 0.045331660832646836, + "maximum_absolute_error": 0.15625 + }, + { + "row": 51, + "rms_error": 0.04472090061403638, + "maximum_absolute_error": 0.16015625 + }, + { + "row": 16, + "rms_error": 0.04423429491541239, + "maximum_absolute_error": 0.21875 + }, + { + "row": 47, + "rms_error": 0.04389095910014947, + "maximum_absolute_error": 0.15625 + } + ], + "worst_columns": [ + { + "column": 8818, + "rms_error": 0.09709327548954391, + "maximum_absolute_error": 0.5 + }, + { + "column": 7986, + "rms_error": 0.09627397740181808, + "maximum_absolute_error": 0.46875 + }, + { + "column": 11622, + "rms_error": 0.08554213785093456, + "maximum_absolute_error": 0.375 + }, + { + "column": 462, + "rms_error": 0.08519213559026834, + "maximum_absolute_error": 0.28125 + }, + { + "column": 3599, + "rms_error": 0.08371685357845365, + "maximum_absolute_error": 0.25 + }, + { + "column": 19235, + "rms_error": 0.08165431451157436, + "maximum_absolute_error": 0.359375 + }, + { + "column": 25491, + "rms_error": 0.07994584762781862, + "maximum_absolute_error": 0.40625 + }, + { + "column": 1492, + "rms_error": 0.07918551781581398, + "maximum_absolute_error": 0.25 + }, + { + "column": 14008, + "rms_error": 0.07815060196044908, + "maximum_absolute_error": 0.35302734375 + }, + { + "column": 11353, + "rms_error": 0.07677527003470581, + "maximum_absolute_error": 0.359375 + }, + { + "column": 3306, + "rms_error": 0.07596382522597174, + "maximum_absolute_error": 0.1875 + }, + { + "column": 4524, + "rms_error": 0.07569792211937734, + "maximum_absolute_error": 0.3125 + }, + { + "column": 89785, + "rms_error": 0.0753367695055099, + "maximum_absolute_error": 0.3515625 + }, + { + "column": 49422, + "rms_error": 0.07523464128064877, + "maximum_absolute_error": 0.1875 + }, + { + "column": 27387, + "rms_error": 0.07408431074043365, + "maximum_absolute_error": 0.32421875 + }, + { + "column": 53113, + "rms_error": 0.07396998564115571, + "maximum_absolute_error": 0.3125 + }, + { + "column": 17606, + "rms_error": 0.0732872628685989, + "maximum_absolute_error": 0.28125 + }, + { + "column": 1745, + "rms_error": 0.072108143964528, + "maximum_absolute_error": 0.30859375 + }, + { + "column": 43577, + "rms_error": 0.07135097749492296, + "maximum_absolute_error": 0.234375 + }, + { + "column": 47479, + "rms_error": 0.07104191403146375, + "maximum_absolute_error": 0.3203125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-8-prose-64", + "phase": "prompt", + "native_logits_sha256": "e4374b6b58d2f8863dbac5560eb80c9e3d8f47c767b8c98d4617884d7700f686", + "reference_logits_sha256": "f21a2db3c896fc517835f98d9647ae7f620b3b6ca8081ad66ed66a368faad2cd", + "token_fixture_sha256": "9c3636ee22fb5d1a9d4d6e3cc0b8e47cce20cb4288bb16f2970de8d55ee27708" + }, + { + "rows": 513, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "relative_rms": 0.00937199471206947, + "maximum_absolute_error": 1.03125, + "differences": [ + { + "row": 5, + "native_top": 440, + "reference_top": 334, + "native_maximum_set": [ + 440 + ], + "reference_maximum_set": [ + 334, + 440 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + }, + { + "row": 33, + "native_top": 1579, + "reference_top": 2925, + "native_maximum_set": [ + 1579, + 2925 + ], + "reference_maximum_set": [ + 2925 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + } + ], + "worst_rows": [ + { + "row": 477, + "rms_error": 0.11058203618233109, + "maximum_absolute_error": 0.25 + }, + { + "row": 461, + "rms_error": 0.10228833277998309, + "maximum_absolute_error": 0.6875 + }, + { + "row": 57, + "rms_error": 0.10170725232544649, + "maximum_absolute_error": 0.41259765625 + }, + { + "row": 109, + "rms_error": 0.09793446656293779, + "maximum_absolute_error": 0.28125 + }, + { + "row": 265, + "rms_error": 0.09069264594409301, + "maximum_absolute_error": 0.22900390625 + }, + { + "row": 453, + "rms_error": 0.08993290478914225, + "maximum_absolute_error": 0.21875 + }, + { + "row": 180, + "rms_error": 0.08623107224559155, + "maximum_absolute_error": 0.25 + }, + { + "row": 422, + "rms_error": 0.08400616582617591, + "maximum_absolute_error": 0.53125 + }, + { + "row": 226, + "rms_error": 0.08302799110176858, + "maximum_absolute_error": 0.21875 + }, + { + "row": 96, + "rms_error": 0.08299528758932778, + "maximum_absolute_error": 0.28125 + }, + { + "row": 160, + "rms_error": 0.08260373100469781, + "maximum_absolute_error": 0.21875 + }, + { + "row": 294, + "rms_error": 0.08245276417184662, + "maximum_absolute_error": 1.03125 + }, + { + "row": 263, + "rms_error": 0.08233848446063262, + "maximum_absolute_error": 0.296875 + }, + { + "row": 334, + "rms_error": 0.08101785659201359, + "maximum_absolute_error": 0.21875 + }, + { + "row": 78, + "rms_error": 0.07880640290699682, + "maximum_absolute_error": 0.21875 + }, + { + "row": 311, + "rms_error": 0.0781917783439388, + "maximum_absolute_error": 0.203125 + }, + { + "row": 399, + "rms_error": 0.07579832492536655, + "maximum_absolute_error": 0.1875 + }, + { + "row": 337, + "rms_error": 0.07461948439029503, + "maximum_absolute_error": 0.171875 + }, + { + "row": 70, + "rms_error": 0.07361931797325384, + "maximum_absolute_error": 0.2890625 + }, + { + "row": 23, + "rms_error": 0.07232874153553556, + "maximum_absolute_error": 0.1875 + } + ], + "worst_columns": [ + { + "column": 334, + "rms_error": 0.09123395835991276, + "maximum_absolute_error": 0.59375 + }, + { + "column": 10625, + "rms_error": 0.0874009612140456, + "maximum_absolute_error": 0.40625 + }, + { + "column": 9798, + "rms_error": 0.07952238098979672, + "maximum_absolute_error": 1.03125 + }, + { + "column": 3498, + "rms_error": 0.0777612919032994, + "maximum_absolute_error": 0.625 + }, + { + "column": 405, + "rms_error": 0.07751982385411463, + "maximum_absolute_error": 0.375 + }, + { + "column": 278, + "rms_error": 0.0756225717974014, + "maximum_absolute_error": 0.6875 + }, + { + "column": 7161, + "rms_error": 0.07545570629705853, + "maximum_absolute_error": 0.3125 + }, + { + "column": 10494, + "rms_error": 0.07503483938607018, + "maximum_absolute_error": 0.34375 + }, + { + "column": 4475, + "rms_error": 0.07390448596090006, + "maximum_absolute_error": 0.625 + }, + { + "column": 229, + "rms_error": 0.07378687217369954, + "maximum_absolute_error": 0.25 + }, + { + "column": 43345, + "rms_error": 0.07338387351302249, + "maximum_absolute_error": 0.34375 + }, + { + "column": 1840, + "rms_error": 0.07280120795143853, + "maximum_absolute_error": 0.46875 + }, + { + "column": 3300, + "rms_error": 0.07278503165797157, + "maximum_absolute_error": 0.53125 + }, + { + "column": 6026, + "rms_error": 0.07268794317470788, + "maximum_absolute_error": 0.375 + }, + { + "column": 3431, + "rms_error": 0.07203168063005132, + "maximum_absolute_error": 0.25 + }, + { + "column": 20, + "rms_error": 0.07201364921827627, + "maximum_absolute_error": 0.3125 + }, + { + "column": 1008, + "rms_error": 0.07200377782882768, + "maximum_absolute_error": 0.5625 + }, + { + "column": 677, + "rms_error": 0.07173890006753607, + "maximum_absolute_error": 0.28125 + }, + { + "column": 8, + "rms_error": 0.07148145421965822, + "maximum_absolute_error": 0.34375 + }, + { + "column": 16420, + "rms_error": 0.07075007746790489, + "maximum_absolute_error": 0.359375 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-8-reasoning-513", + "phase": "prompt", + "native_logits_sha256": "0fbe7f9ff161bf9b0961bd4d64b081f775dc51bb8eb72e7c1bcef7ac9bcbd582", + "reference_logits_sha256": "3abcad4537454a6995fc3a7a75fc74d47a0e148eb0f2e914ee9c46bc1fd9595a", + "token_fixture_sha256": "c8ce58a8d512375ea4635bc85d0ea8a235506aa56026087bda216ea61357a863" + } + ], + "rows": 5581, + "top1_differences": 37, + "disjoint_maximum_sets": 12, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655" +} diff --git a/docs/receipts/lfm3/all-width-performance-baseline.json b/docs/receipts/lfm3/all-width-performance-baseline.json new file mode 100644 index 0000000000..a57af54958 --- /dev/null +++ b/docs/receipts/lfm3/all-width-performance-baseline.json @@ -0,0 +1,13038 @@ +{ + "native_sha256": "d1656249bdb581b147faacbd3b3c557c00b6c45b710eeddce517a67178b6ba2b", + "scope": "shared-attention baseline before greedy SIMD and paired FFN dispatch", + "hardware": "Apple M5 Ultra, 256 GiB, macOS 27.0 (26A428)", + "standard": { + "mlx": "0.32.4", + "mlx_vlm_source": "230fc308efb577d87446306bfb8a3e9f1a129558" + }, + "settings": { + "context": 70000, + "parallel": 4, + "native_prompt_cache_gib": 0, + "standard_APC_ENABLED": 0, + "standard_APC_DISK_ENABLED": 0, + "temperature": 0, + "top_p": 1, + "top_k": 0, + "repetition_penalty": 1, + "thinking": false, + "decode_tokens": 256, + "decode_reps": 3, + "cold_tokens": 16, + "cold_reps": 1 + }, + "cells": [ + { + "family": "text", + "bits": 4, + "engine": "native", + "decode": [ + { + "label": "text-4-native", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 308.4647098580153, + "decode_tps_all": [ + 308.52, + 308.46, + 307.92 + ], + "ttft_s_median": 0.02111462503671646, + "prefill_tps_median": 2510.109, + "itl_mean_ms_median": 3.242, + "itl_p50_ms_median": 3.305, + "itl_p95_ms_median": 3.76, + "round_gap_p50_ms_median": 3.318, + "round_gap_p95_ms_median": 3.76, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ] + }, + { + "label": "text-4-native", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 321.89276348274177, + "decode_tps_all": [ + 321.89, + 322.39, + 321.74 + ], + "ttft_s_median": 0.0213663331232965, + "prefill_tps_median": 2527.341, + "itl_mean_ms_median": 3.107, + "itl_p50_ms_median": 3.315, + "itl_p95_ms_median": 3.768, + "round_gap_p50_ms_median": 3.367, + "round_gap_p95_ms_median": 3.77, + "tokens_per_chunk_median": 1.103, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ] + } + ], + "concurrent": { + "label": "text-4-native", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 309.4, + "per_stream_tps": 309.4, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 302.8393256547563, + "makespan_s": 0.8453327501192689, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 309.7563221133715, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.844289124943316, + "end_to_end_tps": 303.21366512589793, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 308.3742193814178, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8480152082629502, + "end_to_end_tps": 301.881378429973, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 309.36733943815784, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8453327501192689, + "end_to_end_tps": 302.8393256547563, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 676.3, + "per_stream_tps": 174.3, + "ttft_s_max": 0.09, + "timing_valid": true, + "end_to_end_tps": 668.8699238759951, + "makespan_s": 1.5309404167346656, + "steady_tps": 705.3, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 670.508156396887, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5431040828116238, + "end_to_end_tps": 663.5974924868409, + "timing_valid": true, + "steady_tps": 698.1281100809105, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 676.306147573392, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5309404167346656, + "end_to_end_tps": 668.8699238759951, + "timing_valid": true, + "steady_tps": 705.2647451571048, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 732.9706704102962, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.4146986668929458, + "end_to_end_tps": 723.8290555889023, + "timing_valid": true, + "steady_tps": 766.3697135364156, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 322.2, + "per_stream_tps": 322.2, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 314.93200598411573, + "makespan_s": 0.8128738747909665, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 321.8879548797919, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8135802079923451, + "end_to_end_tps": 314.65858864945335, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 322.6958301992876, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8116239998489618, + "end_to_end_tps": 315.4169911777375, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 322.18389309796515, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8128738747909665, + "end_to_end_tps": 314.93200598411573, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 708.4, + "per_stream_tps": 179.2, + "ttft_s_max": 0.09, + "timing_valid": true, + "end_to_end_tps": 699.8200133363089, + "makespan_s": 1.4632333749905229, + "steady_tps": 740.6, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 664.0454774498024, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5577420420013368, + "end_to_end_tps": 657.3617276737282, + "timing_valid": true, + "steady_tps": 691.7322185886699, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 708.3945594222535, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.4632333749905229, + "end_to_end_tps": 699.8200133363089, + "timing_valid": true, + "steady_tps": 740.6092301868067, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 714.1027921137548, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.452291417401284, + "end_to_end_tps": 705.092647199097, + "timing_valid": true, + "steady_tps": 746.5362274902307, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + } + ] + } + ] + }, + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.825507041066885, + "elapsed_s": 0.9079782911576331, + "engine_prefill_s": 0.813756542, + "full_token_sha256": "c244b715e827fd1ce59aa4e249cb47900812abeee1482b0a79a1b251fe0f3262", + "output_text_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "normalized_body_sha256": "b9bc5e2feab4faa89e3cff58510809a260dfe738c75102107e2ccb2a018c3f3e", + "peak_physical_process_footprint_bytes": 1491076776 + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 3.614595875144005, + "elapsed_s": 3.695393417030573, + "engine_prefill_s": 3.601386791, + "full_token_sha256": "3f913af21b98f7168ba1fcd27bd9f16b62982dcf6168b991091bbd6e4d9e7fed", + "output_text_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "normalized_body_sha256": "e4d6edf5df4aa5f7d6633ccf51897404f88b486f5cc919ed353e40acb5aa8ce1", + "peak_physical_process_footprint_bytes": 1629882048 + }, + { + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 20.516632792074233, + "elapsed_s": 20.598936625290662, + "engine_prefill_s": 20.498580167, + "full_token_sha256": "e15e7490e640d938a0a846dfc9de1dd7a51e50ae97ca4fd42a213cbb633f66f5", + "output_text_sha256": "382055acebd5e91b010ffa41b6497ebdb6d22c9bbdd7a2ab220c18df2e11f6b3", + "normalized_body_sha256": "667df3a5602ea177f4c8014d7c12c748d3df65fa026d4891a8dd87977c8c8fcf", + "peak_physical_process_footprint_bytes": 2185103040 + }, + { + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 59.7936379997991, + "elapsed_s": 59.87897541699931, + "engine_prefill_s": 59.772125333, + "full_token_sha256": "cf6c4062b60dc49e0ca503b7be2aaf3d247d73be037d269cf73e127a5e35aa32", + "output_text_sha256": "49b9640f88ffd97f99e1c6a354211b68d7b226874d3d8c1afbc261003f02cb39", + "normalized_body_sha256": "f82e80f89b846734ed458943f26d45bac00bfdae2fe2c20e0680ef1060e490ca", + "peak_physical_process_footprint_bytes": 2925315776 + } + ] + }, + { + "family": "text", + "bits": 4, + "engine": "standard", + "decode": [ + { + "label": "text-4-standard", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 345.11249650803427, + "decode_tps_all": [ + 344.67, + 345.18, + 345.11 + ], + "ttft_s_median": 0.017844750080257654, + "prefill_tps_median": 2970.061, + "itl_mean_ms_median": 2.898, + "itl_p50_ms_median": 2.877, + "itl_p95_ms_median": 2.953, + "round_gap_p50_ms_median": 2.877, + "round_gap_p95_ms_median": 2.953, + "tokens_per_chunk_median": 1.0, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ] + }, + { + "label": "text-4-standard", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 344.8503001737025, + "decode_tps_all": [ + 344.96, + 344.78, + 344.85 + ], + "ttft_s_median": 0.020470124669373035, + "prefill_tps_median": 2637.991, + "itl_mean_ms_median": 2.9, + "itl_p50_ms_median": 2.889, + "itl_p95_ms_median": 2.989, + "round_gap_p50_ms_median": 2.889, + "round_gap_p95_ms_median": 2.989, + "tokens_per_chunk_median": 1.004, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "fc6c1799b10d5ece11495fb9eb854a5562db013147b5382e1e92d4dc95bf5ae7" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "fc6c1799b10d5ece11495fb9eb854a5562db013147b5382e1e92d4dc95bf5ae7" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "fc6c1799b10d5ece11495fb9eb854a5562db013147b5382e1e92d4dc95bf5ae7" + } + ] + } + ], + "concurrent": { + "label": "text-4-standard", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 343.6, + "per_stream_tps": 343.6, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 336.8239712951299, + "makespan_s": 0.760040916968137, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 343.3809482175411, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.760858042165637, + "end_to_end_tps": 336.4622384372056, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 343.97298383311994, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7594917919486761, + "end_to_end_tps": 337.0675005494985, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 343.64382729149696, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.760040916968137, + "end_to_end_tps": 336.8239712951299, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 1001.0, + "per_stream_tps": 253.6, + "ttft_s_max": 0.24, + "timing_valid": true, + "end_to_end_tps": 973.0719675296708, + "makespan_s": 1.0523373750038445, + "steady_tps": 1022.8, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 918.8671462139706, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.3123167087323964, + "end_to_end_tps": 780.2994453900617, + "timing_valid": true, + "steady_tps": 981.721230409707, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 1001.0453319745648, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.0523373750038445, + "end_to_end_tps": 973.0719675296708, + "timing_valid": true, + "steady_tps": 1022.8029634137688, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 1001.4474764724264, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.0502625000663102, + "end_to_end_tps": 974.9943465898745, + "timing_valid": true, + "steady_tps": 1023.2139071082958, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 346.5, + "per_stream_tps": 346.5, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 338.30775272524164, + "makespan_s": 0.7567074592225254, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 346.5036425154079, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7565196668729186, + "end_to_end_tps": 338.39173151727636, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "fc6c1799b10d5ece11495fb9eb854a5562db013147b5382e1e92d4dc95bf5ae7" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 346.45893799086025, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7567074592225254, + "end_to_end_tps": 338.30775272524164, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "fc6c1799b10d5ece11495fb9eb854a5562db013147b5382e1e92d4dc95bf5ae7" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 345.1847709977767, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7592205828987062, + "end_to_end_tps": 337.18790792340127, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "fc6c1799b10d5ece11495fb9eb854a5562db013147b5382e1e92d4dc95bf5ae7" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 1011.9, + "per_stream_tps": 255.1, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 972.4176592870928, + "makespan_s": 1.053045458626002, + "steady_tps": 1023.2, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 1011.5557018664346, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.0538123338483274, + "end_to_end_tps": 971.7100162042531, + "timing_valid": true, + "steady_tps": 1022.8554126011608, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "24f4feefb469f86e1a3fc752cbd6d2404c1530c3c8f87fbf0408b1741da5017a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f3e436a1e2277cc5f0ba02cc9de357c6286ea1a8d1c78cb277ec1629464af7b" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f3e436a1e2277cc5f0ba02cc9de357c6286ea1a8d1c78cb277ec1629464af7b" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f3e436a1e2277cc5f0ba02cc9de357c6286ea1a8d1c78cb277ec1629464af7b" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 1012.145204650242, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.0528882499784231, + "end_to_end_tps": 972.5628527253342, + "timing_valid": true, + "steady_tps": 1023.5602836439078, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f3e436a1e2277cc5f0ba02cc9de357c6286ea1a8d1c78cb277ec1629464af7b" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f3e436a1e2277cc5f0ba02cc9de357c6286ea1a8d1c78cb277ec1629464af7b" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f3e436a1e2277cc5f0ba02cc9de357c6286ea1a8d1c78cb277ec1629464af7b" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "24f4feefb469f86e1a3fc752cbd6d2404c1530c3c8f87fbf0408b1741da5017a" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 1011.8922139792778, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.053045458626002, + "end_to_end_tps": 972.4176592870928, + "timing_valid": true, + "steady_tps": 1023.2198752799725, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f3e436a1e2277cc5f0ba02cc9de357c6286ea1a8d1c78cb277ec1629464af7b" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f3e436a1e2277cc5f0ba02cc9de357c6286ea1a8d1c78cb277ec1629464af7b" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f3e436a1e2277cc5f0ba02cc9de357c6286ea1a8d1c78cb277ec1629464af7b" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "24f4feefb469f86e1a3fc752cbd6d2404c1530c3c8f87fbf0408b1741da5017a" + } + ] + } + ] + } + ] + }, + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.1617831252515316, + "elapsed_s": 0.20617104228585958, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "normalized_body_sha256": "b9bc5e2feab4faa89e3cff58510809a260dfe738c75102107e2ccb2a018c3f3e", + "peak_physical_process_footprint_bytes": 2587510944 + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.738223166204989, + "elapsed_s": 0.7843746249563992, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "normalized_body_sha256": "e4d6edf5df4aa5f7d6633ccf51897404f88b486f5cc919ed353e40acb5aa8ce1", + "peak_physical_process_footprint_bytes": 2904115336 + }, + { + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 1.6866859588772058, + "elapsed_s": 1.7399688339792192, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "5bb9e62f6a8c22891e1f0f3814891df33bf136e0e41a2f3e531f4cbf2af171a2", + "normalized_body_sha256": "667df3a5602ea177f4c8014d7c12c748d3df65fa026d4891a8dd87977c8c8fcf", + "peak_physical_process_footprint_bytes": 3534162152 + }, + { + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 4.055460750125349, + "elapsed_s": 4.124586749821901, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "49b9640f88ffd97f99e1c6a354211b68d7b226874d3d8c1afbc261003f02cb39", + "normalized_body_sha256": "f82e80f89b846734ed458943f26d45bac00bfdae2fe2c20e0680ef1060e490ca", + "peak_physical_process_footprint_bytes": 4395944072 + } + ] + }, + { + "family": "text", + "bits": 6, + "engine": "native", + "decode": [ + { + "label": "text-6-native", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 266.67554408061716, + "decode_tps_all": [ + 266.68, + 266.83, + 266.42 + ], + "ttft_s_median": 0.023621333297342062, + "prefill_tps_median": 2243.734, + "itl_mean_ms_median": 3.75, + "itl_p50_ms_median": 3.768, + "itl_p95_ms_median": 4.222, + "round_gap_p50_ms_median": 3.768, + "round_gap_p95_ms_median": 4.222, + "tokens_per_chunk_median": 1.0, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + } + ] + }, + { + "label": "text-6-native", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 266.21102253383594, + "decode_tps_all": [ + 266.21, + 266.28, + 265.56 + ], + "ttft_s_median": 0.0237058331258595, + "prefill_tps_median": 2277.92, + "itl_mean_ms_median": 3.756, + "itl_p50_ms_median": 3.771, + "itl_p95_ms_median": 4.238, + "round_gap_p50_ms_median": 3.771, + "round_gap_p95_ms_median": 4.238, + "tokens_per_chunk_median": 1.0, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + } + ] + } + ], + "concurrent": { + "label": "text-6-native", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 266.4, + "per_stream_tps": 266.4, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 261.0284853173837, + "makespan_s": 0.9807358752004802, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 266.0430014502786, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9821044160053134, + "end_to_end_tps": 260.66474789032515, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 266.41316200723736, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9807358752004802, + "end_to_end_tps": 261.0284853173837, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 266.66830514219134, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9798028329387307, + "end_to_end_tps": 261.2770563565091, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 602.8, + "per_stream_tps": 154.3, + "ttft_s_max": 0.09, + "timing_valid": true, + "end_to_end_tps": 596.242160524973, + "makespan_s": 1.717422999907285, + "steady_tps": 626.4, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 611.7912724925352, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.691136541776359, + "end_to_end_tps": 605.5099483122734, + "timing_valid": true, + "steady_tps": 636.0770843205082, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 597.5681174207766, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.7322917920537293, + "end_to_end_tps": 591.1244310555732, + "timing_valid": true, + "steady_tps": 620.7943247525583, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 602.7865431398862, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.717422999907285, + "end_to_end_tps": 596.242160524973, + "timing_valid": true, + "steady_tps": 626.4333561080279, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 265.5, + "per_stream_tps": 265.5, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 260.08098512543006, + "makespan_s": 0.9843087908811867, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 265.4762572283853, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9843087908811867, + "end_to_end_tps": 260.08098512543006, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 264.76269399580275, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9869117080233991, + "end_to_end_tps": 259.39503799455423, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 265.57941136497345, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9839359167963266, + "end_to_end_tps": 260.1795458727945, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 615.4, + "per_stream_tps": 155.9, + "ttft_s_max": 0.1, + "timing_valid": true, + "end_to_end_tps": 608.4127620296283, + "makespan_s": 1.683067916892469, + "steady_tps": 643.5, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 597.41726854559, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.731481125112623, + "end_to_end_tps": 591.4011912393181, + "timing_valid": true, + "steady_tps": 623.9719845346317, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 621.4545767949363, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.6673438330180943, + "end_to_end_tps": 614.1504707798847, + "timing_valid": true, + "steady_tps": 650.329914564971, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 615.4444425698488, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.683067916892469, + "end_to_end_tps": 608.4127620296283, + "timing_valid": true, + "steady_tps": 643.478836927162, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + } + } + ] + } + ] + } + ] + }, + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.9109433749690652, + "elapsed_s": 1.000833542086184, + "engine_prefill_s": 0.899962584, + "full_token_sha256": "b482722716a27c86c613d49d437feba1fe030ed496fb1141b0498ac017ee32c1", + "output_text_sha256": "6cb715bc54d1b0adbe24d918f8f147ac719d55d25abea6c65bc204eb159bebe8", + "normalized_body_sha256": "b9bc5e2feab4faa89e3cff58510809a260dfe738c75102107e2ccb2a018c3f3e", + "peak_physical_process_footprint_bytes": 1923352568 + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 3.964558834210038, + "elapsed_s": 4.052450334187597, + "engine_prefill_s": 3.952161833, + "full_token_sha256": "06efc9b74a3d0b1c7f8b7e69ae2da268963d81d498818ac39f495593a6c6da42", + "output_text_sha256": "63b1d289496206389642e98d3a58dffa5a753c8e5e32a1c40ced27643265c352", + "normalized_body_sha256": "e4d6edf5df4aa5f7d6633ccf51897404f88b486f5cc919ed353e40acb5aa8ce1", + "peak_physical_process_footprint_bytes": 2062190584 + }, + { + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 21.88658033311367, + "elapsed_s": 21.976169290952384, + "engine_prefill_s": 21.870679584, + "full_token_sha256": "2515ee177085220869a02ccad4a3dee8b4f6e94a8e8128cc49057f61fa53530a", + "output_text_sha256": "92d18eaba9b6974d02286b179a5981ebffd01ab62f9eb2840707ca3fd372193f", + "normalized_body_sha256": "667df3a5602ea177f4c8014d7c12c748d3df65fa026d4891a8dd87977c8c8fcf", + "peak_physical_process_footprint_bytes": 2617345992 + }, + { + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 62.529848916921765, + "elapsed_s": 62.6223569996655, + "engine_prefill_s": 62.508319042, + "full_token_sha256": "3c95d1174f8902a648ae88c4f20e5bd8f44a1d4dc240a44bc4e0ca29404bfd8e", + "output_text_sha256": "1307bc27c5739d7a4c363433733092e2f728d436553a65ee1f0e0eb8ce05d5a6", + "normalized_body_sha256": "f82e80f89b846734ed458943f26d45bac00bfdae2fe2c20e0680ef1060e490ca", + "peak_physical_process_footprint_bytes": 3358001144 + } + ] + }, + { + "family": "text", + "bits": 6, + "engine": "standard", + "decode": [ + { + "label": "text-6-standard", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 288.77684348821685, + "decode_tps_all": [ + 288.47, + 288.84, + 288.78 + ], + "ttft_s_median": 0.018470000009983778, + "prefill_tps_median": 2869.518, + "itl_mean_ms_median": 3.463, + "itl_p50_ms_median": 3.44, + "itl_p95_ms_median": 3.483, + "round_gap_p50_ms_median": 3.44, + "round_gap_p95_ms_median": 3.483, + "tokens_per_chunk_median": 1.0, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "2acb42920e45eb8b8048e4fac1167a60dea14112244ff37fa4465f51ff66cd69" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "2acb42920e45eb8b8048e4fac1167a60dea14112244ff37fa4465f51ff66cd69" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "2acb42920e45eb8b8048e4fac1167a60dea14112244ff37fa4465f51ff66cd69" + } + ] + }, + { + "label": "text-6-standard", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 288.17289019180345, + "decode_tps_all": [ + 288.59, + 287.72, + 288.17 + ], + "ttft_s_median": 0.018647165969014168, + "prefill_tps_median": 2895.882, + "itl_mean_ms_median": 3.47, + "itl_p50_ms_median": 3.448, + "itl_p95_ms_median": 3.497, + "round_gap_p50_ms_median": 3.448, + "round_gap_p95_ms_median": 3.497, + "tokens_per_chunk_median": 1.0, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + } + ] + } + ], + "concurrent": { + "label": "text-6-standard", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 284.3, + "per_stream_tps": 284.3, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 279.15401069124334, + "makespan_s": 0.9170564999803901, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 283.79959244179173, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9191879173740745, + "end_to_end_tps": 278.50670701953726, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "2acb42920e45eb8b8048e4fac1167a60dea14112244ff37fa4465f51ff66cd69" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 284.60684364228086, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9160721669904888, + "end_to_end_tps": 279.4539657732642, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "2acb42920e45eb8b8048e4fac1167a60dea14112244ff37fa4465f51ff66cd69" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 284.3352993073837, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9170564999803901, + "end_to_end_tps": 279.15401069124334, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "2acb42920e45eb8b8048e4fac1167a60dea14112244ff37fa4465f51ff66cd69" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 921.0, + "per_stream_tps": 234.2, + "ttft_s_max": 0.06, + "timing_valid": true, + "end_to_end_tps": 897.0642495705168, + "makespan_s": 1.1415012921206653, + "steady_tps": 936.9, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 918.072252558559, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1475952919572592, + "end_to_end_tps": 892.3006282585357, + "timing_valid": true, + "steady_tps": 936.775305717054, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "f30c4fcc66bb085cd245495466aed8b6f4cc69bc70f9130228b494fb14692c0b" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "f30c4fcc66bb085cd245495466aed8b6f4cc69bc70f9130228b494fb14692c0b" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "f30c4fcc66bb085cd245495466aed8b6f4cc69bc70f9130228b494fb14692c0b" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "861ba93f9d4752f8cfc26b981a8428d0db4f2d30729fca317935d9aefcb04d31" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 920.9967972794149, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1415012921206653, + "end_to_end_tps": 897.0642495705168, + "timing_valid": true, + "steady_tps": 936.9416729615583, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "861ba93f9d4752f8cfc26b981a8428d0db4f2d30729fca317935d9aefcb04d31" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "f30c4fcc66bb085cd245495466aed8b6f4cc69bc70f9130228b494fb14692c0b" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "f30c4fcc66bb085cd245495466aed8b6f4cc69bc70f9130228b494fb14692c0b" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "f30c4fcc66bb085cd245495466aed8b6f4cc69bc70f9130228b494fb14692c0b" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 921.3576115323501, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1409097909927368, + "end_to_end_tps": 897.5293297369195, + "timing_valid": true, + "steady_tps": 937.0439594056941, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "f30c4fcc66bb085cd245495466aed8b6f4cc69bc70f9130228b494fb14692c0b" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "f30c4fcc66bb085cd245495466aed8b6f4cc69bc70f9130228b494fb14692c0b" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "861ba93f9d4752f8cfc26b981a8428d0db4f2d30729fca317935d9aefcb04d31" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "f30c4fcc66bb085cd245495466aed8b6f4cc69bc70f9130228b494fb14692c0b" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 288.5, + "per_stream_tps": 288.5, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 283.5146867164152, + "makespan_s": 0.9029514589346945, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 287.65939796325324, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9053023750893772, + "end_to_end_tps": 282.7784473389082, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 288.58160370220327, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9029514589346945, + "end_to_end_tps": 283.5146867164152, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 288.5062366581396, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9028852917253971, + "end_to_end_tps": 283.5354638580818, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 923.0, + "per_stream_tps": 234.4, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 898.8954073721759, + "makespan_s": 1.1391759170219302, + "steady_tps": 937.8, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 918.9221965592948, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1440925002098083, + "end_to_end_tps": 895.0325256150309, + "timing_valid": true, + "steady_tps": 933.9257344347969, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "3eb59d87c7f69b26cdbe4a758521e66b9c5e7af5695f594e4f21e1b3568a41a5" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "47d2eec82471e59cfaeee81c102987be9d4e9f756e8f776f4d7d15f3495187f8" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "3eb59d87c7f69b26cdbe4a758521e66b9c5e7af5695f594e4f21e1b3568a41a5" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "3eb59d87c7f69b26cdbe4a758521e66b9c5e7af5695f594e4f21e1b3568a41a5" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 922.9590122956736, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1391759170219302, + "end_to_end_tps": 898.8954073721759, + "timing_valid": true, + "steady_tps": 937.7828936978541, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "3eb59d87c7f69b26cdbe4a758521e66b9c5e7af5695f594e4f21e1b3568a41a5" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "3eb59d87c7f69b26cdbe4a758521e66b9c5e7af5695f594e4f21e1b3568a41a5" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "3eb59d87c7f69b26cdbe4a758521e66b9c5e7af5695f594e4f21e1b3568a41a5" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "47d2eec82471e59cfaeee81c102987be9d4e9f756e8f776f4d7d15f3495187f8" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 923.0403775355038, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1390468329191208, + "end_to_end_tps": 898.997275973033, + "timing_valid": true, + "steady_tps": 937.9466907305982, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "47d2eec82471e59cfaeee81c102987be9d4e9f756e8f776f4d7d15f3495187f8" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "3eb59d87c7f69b26cdbe4a758521e66b9c5e7af5695f594e4f21e1b3568a41a5" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "3eb59d87c7f69b26cdbe4a758521e66b9c5e7af5695f594e4f21e1b3568a41a5" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "3eb59d87c7f69b26cdbe4a758521e66b9c5e7af5695f594e4f21e1b3568a41a5" + } + ] + } + ] + } + ] + }, + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.1465860833413899, + "elapsed_s": 0.20005200011655688, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "normalized_body_sha256": "b9bc5e2feab4faa89e3cff58510809a260dfe738c75102107e2ccb2a018c3f3e", + "peak_physical_process_footprint_bytes": 2955479392 + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.41900195786729455, + "elapsed_s": 0.47467212472110987, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "63b1d289496206389642e98d3a58dffa5a753c8e5e32a1c40ced27643265c352", + "normalized_body_sha256": "e4d6edf5df4aa5f7d6633ccf51897404f88b486f5cc919ed353e40acb5aa8ce1", + "peak_physical_process_footprint_bytes": 3205122448 + }, + { + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 1.7611606661230326, + "elapsed_s": 1.826868874952197, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "92d18eaba9b6974d02286b179a5981ebffd01ab62f9eb2840707ca3fd372193f", + "normalized_body_sha256": "667df3a5602ea177f4c8014d7c12c748d3df65fa026d4891a8dd87977c8c8fcf", + "peak_physical_process_footprint_bytes": 3915188648 + }, + { + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 4.2018702500499785, + "elapsed_s": 4.278072207700461, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "e0f28568f3cd872890c0ddfd6c03ed906bfebd320ff050b7236941c1cfe9ba36", + "normalized_body_sha256": "f82e80f89b846734ed458943f26d45bac00bfdae2fe2c20e0680ef1060e490ca", + "peak_physical_process_footprint_bytes": 4834953568 + } + ] + }, + { + "family": "text", + "bits": 8, + "engine": "native", + "decode": [ + { + "label": "text-8-native", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 253.18790252186744, + "decode_tps_all": [ + 254.08, + 253.19, + 252.48 + ], + "ttft_s_median": 0.02083920780569315, + "prefill_tps_median": 2543.283, + "itl_mean_ms_median": 3.95, + "itl_p50_ms_median": 3.953, + "itl_p95_ms_median": 4.431, + "round_gap_p50_ms_median": 3.953, + "round_gap_p95_ms_median": 4.431, + "tokens_per_chunk_median": 1.0, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + } + ] + }, + { + "label": "text-8-native", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 251.15126381972917, + "decode_tps_all": [ + 250.66, + 251.49, + 251.15 + ], + "ttft_s_median": 0.021045291796326637, + "prefill_tps_median": 2565.895, + "itl_mean_ms_median": 3.982, + "itl_p50_ms_median": 3.973, + "itl_p95_ms_median": 4.428, + "round_gap_p50_ms_median": 3.973, + "round_gap_p95_ms_median": 4.428, + "tokens_per_chunk_median": 1.0, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + } + ] + } + ], + "concurrent": { + "label": "text-8-native", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 253.0, + "per_stream_tps": 253.0, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 248.80167007039364, + "makespan_s": 1.0289320000447333, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 252.96254487417298, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.0288784159347415, + "end_to_end_tps": 248.8146276908945, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 252.95948135832074, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.0289320000447333, + "end_to_end_tps": 248.80167007039364, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 252.85634704459696, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.029303917195648, + "end_to_end_tps": 248.71177086110328, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 615.3, + "per_stream_tps": 157.8, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 609.7850402331, + "makespan_s": 1.6792802913114429, + "steady_tps": 636.4, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 615.3157673376985, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.6792802913114429, + "end_to_end_tps": 609.7850402331, + "timing_valid": true, + "steady_tps": 636.4322178106547, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 615.1452686112303, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.6814224170520902, + "end_to_end_tps": 609.0081764196419, + "timing_valid": true, + "steady_tps": 636.2998279369203, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 631.9270291055683, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.6370734167285264, + "end_to_end_tps": 625.5064614306228, + "timing_valid": true, + "steady_tps": 654.8717250363792, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 250.7, + "per_stream_tps": 250.7, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 246.5730599718849, + "makespan_s": 1.0382318329066038, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 250.68866655940016, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.0383407091721892, + "end_to_end_tps": 246.54720530421505, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 251.1484500812127, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.0364424581639469, + "end_to_end_tps": 246.99875809169652, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 250.70870290908798, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.0382318329066038, + "end_to_end_tps": 246.5730599718849, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 629.8, + "per_stream_tps": 160.4, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 623.8642704902273, + "makespan_s": 1.6413826667703688, + "steady_tps": 656.2, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 629.7772257130378, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.6413826667703688, + "end_to_end_tps": 623.8642704902273, + "timing_valid": true, + "steady_tps": 656.2113181767336, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 615.0704622303156, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.6814822079613805, + "end_to_end_tps": 608.9865210298549, + "timing_valid": true, + "steady_tps": 640.1659938169346, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 644.7636489781453, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.6054776245728135, + "end_to_end_tps": 637.8164256711249, + "timing_valid": true, + "steady_tps": 672.3520703113089, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + } + } + ] + } + ] + } + ] + }, + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.8088387078605592, + "elapsed_s": 0.9010657081380486, + "engine_prefill_s": 0.797375333, + "full_token_sha256": "06efc9b74a3d0b1c7f8b7e69ae2da268963d81d498818ac39f495593a6c6da42", + "output_text_sha256": "63b1d289496206389642e98d3a58dffa5a753c8e5e32a1c40ced27643265c352", + "normalized_body_sha256": "b9bc5e2feab4faa89e3cff58510809a260dfe738c75102107e2ccb2a018c3f3e", + "peak_physical_process_footprint_bytes": 2355841304 + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 3.563282791990787, + "elapsed_s": 3.6540214172564447, + "engine_prefill_s": 3.551555125, + "full_token_sha256": "711bccf584b7cba05cf72e92b63640c9025fc1ba4468767322fe65278e9b91bd", + "output_text_sha256": "f6fe0f139e7e212c4558b810f2bc93e49dbe1b63c81c9c14759f5eaf0e3952a6", + "normalized_body_sha256": "e4d6edf5df4aa5f7d6633ccf51897404f88b486f5cc919ed353e40acb5aa8ce1", + "peak_physical_process_footprint_bytes": 2494613736 + }, + { + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 20.21945449989289, + "elapsed_s": 20.311934791970998, + "engine_prefill_s": 20.201873208, + "full_token_sha256": "8a12db93a7da210cc96363a86a0f00ef604f06424be764ce6fca5c4ff0e8a2b3", + "output_text_sha256": "505723e789a004173b6147f2a302d21a5724ec0341c9505f0577c7052879ca10", + "normalized_body_sha256": "667df3a5602ea177f4c8014d7c12c748d3df65fa026d4891a8dd87977c8c8fcf", + "peak_physical_process_footprint_bytes": 3049687296 + }, + { + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 59.21170145832002, + "elapsed_s": 59.30652204202488, + "engine_prefill_s": 59.189552875, + "full_token_sha256": "29dc052337ab3ed5b82ea5ef42dbe4c7dc68249617b3b166fe015be590755796", + "output_text_sha256": "e0f28568f3cd872890c0ddfd6c03ed906bfebd320ff050b7236941c1cfe9ba36", + "normalized_body_sha256": "f82e80f89b846734ed458943f26d45bac00bfdae2fe2c20e0680ef1060e490ca", + "peak_physical_process_footprint_bytes": 3789998336 + } + ] + }, + { + "family": "text", + "bits": 8, + "engine": "standard", + "decode": [ + { + "label": "text-8-standard", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 268.50234519894093, + "decode_tps_all": [ + 273.1, + 266.47, + 268.5 + ], + "ttft_s_median": 0.0190572920255363, + "prefill_tps_median": 2781.088, + "itl_mean_ms_median": 3.724, + "itl_p50_ms_median": 3.694, + "itl_p95_ms_median": 3.844, + "round_gap_p50_ms_median": 3.694, + "round_gap_p95_ms_median": 3.844, + "tokens_per_chunk_median": 1.0, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "46a5c50d8ea4d0e89805969b4d818de47e1ca0943bc8f6b63248ed4e6be47ad4" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "46a5c50d8ea4d0e89805969b4d818de47e1ca0943bc8f6b63248ed4e6be47ad4" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "46a5c50d8ea4d0e89805969b4d818de47e1ca0943bc8f6b63248ed4e6be47ad4" + } + ] + }, + { + "label": "text-8-standard", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 268.1664475886384, + "decode_tps_all": [ + 268.17, + 267.61, + 268.21 + ], + "ttft_s_median": 0.02041483297944069, + "prefill_tps_median": 2645.136, + "itl_mean_ms_median": 3.729, + "itl_p50_ms_median": 3.694, + "itl_p95_ms_median": 3.861, + "round_gap_p50_ms_median": 3.694, + "round_gap_p95_ms_median": 3.861, + "tokens_per_chunk_median": 1.0, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "63a2be4ed8a8b152de59da48ae29e10f3db4de57316d186c89c54f5b24551c57" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "63a2be4ed8a8b152de59da48ae29e10f3db4de57316d186c89c54f5b24551c57" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "63a2be4ed8a8b152de59da48ae29e10f3db4de57316d186c89c54f5b24551c57" + } + ] + } + ], + "concurrent": { + "label": "text-8-standard", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 268.5, + "per_stream_tps": 268.5, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 264.25090422566757, + "makespan_s": 0.9687762497924268, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 272.87731875120386, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9537330842576921, + "end_to_end_tps": 268.4189153396618, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "46a5c50d8ea4d0e89805969b4d818de47e1ca0943bc8f6b63248ed4e6be47ad4" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 268.5191563168188, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9687762497924268, + "end_to_end_tps": 264.25090422566757, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "46a5c50d8ea4d0e89805969b4d818de47e1ca0943bc8f6b63248ed4e6be47ad4" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 268.5359109403195, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9703731671907008, + "end_to_end_tps": 263.8160335174335, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "46a5c50d8ea4d0e89805969b4d818de47e1ca0943bc8f6b63248ed4e6be47ad4" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 917.6, + "per_stream_tps": 233.1, + "ttft_s_max": 0.06, + "timing_valid": true, + "end_to_end_tps": 893.8257825602901, + "makespan_s": 1.1456371252425015, + "steady_tps": 933.7, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 913.0404667672308, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1553119579330087, + "end_to_end_tps": 886.3406917661084, + "timing_valid": true, + "steady_tps": 930.7782151133966, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "feaa37c2b89ff1af70ccf2ea98b7c8785133038c2a862c0cf8a7514e66ca6ee3" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "feaa37c2b89ff1af70ccf2ea98b7c8785133038c2a862c0cf8a7514e66ca6ee3" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "f109defe08d7743ac95fae3d57a600d065dc37088301dd61b76a946176c20c69" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "feaa37c2b89ff1af70ccf2ea98b7c8785133038c2a862c0cf8a7514e66ca6ee3" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 918.139534543325, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1448741252534091, + "end_to_end_tps": 894.4214716821776, + "timing_valid": true, + "steady_tps": 933.9742964606187, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "feaa37c2b89ff1af70ccf2ea98b7c8785133038c2a862c0cf8a7514e66ca6ee3" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "feaa37c2b89ff1af70ccf2ea98b7c8785133038c2a862c0cf8a7514e66ca6ee3" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "feaa37c2b89ff1af70ccf2ea98b7c8785133038c2a862c0cf8a7514e66ca6ee3" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "f109defe08d7743ac95fae3d57a600d065dc37088301dd61b76a946176c20c69" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 917.5738328676409, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1456371252425015, + "end_to_end_tps": 893.8257825602901, + "timing_valid": true, + "steady_tps": 933.6564288060797, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "feaa37c2b89ff1af70ccf2ea98b7c8785133038c2a862c0cf8a7514e66ca6ee3" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "feaa37c2b89ff1af70ccf2ea98b7c8785133038c2a862c0cf8a7514e66ca6ee3" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "f109defe08d7743ac95fae3d57a600d065dc37088301dd61b76a946176c20c69" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "feaa37c2b89ff1af70ccf2ea98b7c8785133038c2a862c0cf8a7514e66ca6ee3" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 268.9, + "per_stream_tps": 268.9, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 264.58198119036877, + "makespan_s": 0.9675639998167753, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 273.47436720276283, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9513842076994479, + "end_to_end_tps": 269.08161595307143, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "63a2be4ed8a8b152de59da48ae29e10f3db4de57316d186c89c54f5b24551c57" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 268.8902188233368, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9675639998167753, + "end_to_end_tps": 264.58198119036877, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "63a2be4ed8a8b152de59da48ae29e10f3db4de57316d186c89c54f5b24551c57" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 268.7701355555473, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9696711669676006, + "end_to_end_tps": 264.0070249799989, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "63a2be4ed8a8b152de59da48ae29e10f3db4de57316d186c89c54f5b24551c57" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 919.4, + "per_stream_tps": 233.7, + "ttft_s_max": 0.06, + "timing_valid": true, + "end_to_end_tps": 895.2977085382888, + "makespan_s": 1.1437536254525185, + "steady_tps": 935.1, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 916.6934596679747, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1504525002092123, + "end_to_end_tps": 890.0845535246204, + "timing_valid": true, + "steady_tps": 933.0663223810886, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "cce3d09b123738b0b8e4e4f55beaf42bb6e17ed1a0ddd2136a87a016420e0971" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "cce3d09b123738b0b8e4e4f55beaf42bb6e17ed1a0ddd2136a87a016420e0971" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "cce3d09b123738b0b8e4e4f55beaf42bb6e17ed1a0ddd2136a87a016420e0971" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "cce3d09b123738b0b8e4e4f55beaf42bb6e17ed1a0ddd2136a87a016420e0971" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 919.6505655593013, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.143473165575415, + "end_to_end_tps": 895.5172983746462, + "timing_valid": true, + "steady_tps": 935.5265188913082, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "cce3d09b123738b0b8e4e4f55beaf42bb6e17ed1a0ddd2136a87a016420e0971" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "cce3d09b123738b0b8e4e4f55beaf42bb6e17ed1a0ddd2136a87a016420e0971" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "cce3d09b123738b0b8e4e4f55beaf42bb6e17ed1a0ddd2136a87a016420e0971" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "cce3d09b123738b0b8e4e4f55beaf42bb6e17ed1a0ddd2136a87a016420e0971" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 919.447912299261, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1437536254525185, + "end_to_end_tps": 895.2977085382888, + "timing_valid": true, + "steady_tps": 935.1428541735684, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "cce3d09b123738b0b8e4e4f55beaf42bb6e17ed1a0ddd2136a87a016420e0971" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "cce3d09b123738b0b8e4e4f55beaf42bb6e17ed1a0ddd2136a87a016420e0971" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "cce3d09b123738b0b8e4e4f55beaf42bb6e17ed1a0ddd2136a87a016420e0971" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "cce3d09b123738b0b8e4e4f55beaf42bb6e17ed1a0ddd2136a87a016420e0971" + } + ] + } + ] + } + ] + }, + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.14604083402082324, + "elapsed_s": 0.20214075036346912, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "63b1d289496206389642e98d3a58dffa5a753c8e5e32a1c40ced27643265c352", + "normalized_body_sha256": "b9bc5e2feab4faa89e3cff58510809a260dfe738c75102107e2ccb2a018c3f3e", + "peak_physical_process_footprint_bytes": 3380038272 + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.4110664171166718, + "elapsed_s": 0.46922466717660427, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "f6fe0f139e7e212c4558b810f2bc93e49dbe1b63c81c9c14759f5eaf0e3952a6", + "normalized_body_sha256": "e4d6edf5df4aa5f7d6633ccf51897404f88b486f5cc919ed353e40acb5aa8ce1", + "peak_physical_process_footprint_bytes": 3639429880 + }, + { + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 1.741419083904475, + "elapsed_s": 1.8111548339948058, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "e0f28568f3cd872890c0ddfd6c03ed906bfebd320ff050b7236941c1cfe9ba36", + "normalized_body_sha256": "667df3a5602ea177f4c8014d7c12c748d3df65fa026d4891a8dd87977c8c8fcf", + "peak_physical_process_footprint_bytes": 4340435728 + }, + { + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 4.176876875106245, + "elapsed_s": 4.257043291814625, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "e0f28568f3cd872890c0ddfd6c03ed906bfebd320ff050b7236941c1cfe9ba36", + "normalized_body_sha256": "f82e80f89b846734ed458943f26d45bac00bfdae2fe2c20e0680ef1060e490ca", + "peak_physical_process_footprint_bytes": 5258300200 + } + ] + }, + { + "family": "vl", + "bits": 4, + "engine": "native", + "decode": [ + { + "label": "vl-4-native", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 292.6783216028834, + "decode_tps_all": [ + 292.65, + 293.07, + 292.68 + ], + "ttft_s_median": 0.02120216703042388, + "prefill_tps_median": 2499.744, + "itl_mean_ms_median": 3.417, + "itl_p50_ms_median": 3.456, + "itl_p95_ms_median": 3.958, + "round_gap_p50_ms_median": 3.478, + "round_gap_p95_ms_median": 3.958, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ] + }, + { + "label": "vl-4-native", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 291.52102528407875, + "decode_tps_all": [ + 291.7, + 291.52, + 291.39 + ], + "ttft_s_median": 0.021518750116229057, + "prefill_tps_median": 2509.439, + "itl_mean_ms_median": 3.43, + "itl_p50_ms_median": 3.503, + "itl_p95_ms_median": 3.945, + "round_gap_p50_ms_median": 3.521, + "round_gap_p95_ms_median": 3.945, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ] + } + ], + "concurrent": { + "label": "vl-4-native", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 293.4, + "per_stream_tps": 293.4, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 287.5114327211394, + "makespan_s": 0.8903993749991059, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 293.83489726554734, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8890331671573222, + "end_to_end_tps": 287.953261427308, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 293.374201402275, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8903993749991059, + "end_to_end_tps": 287.5114327211394, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 292.5462931760241, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8928427500650287, + "end_to_end_tps": 286.7246219800236, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 666.8, + "per_stream_tps": 169.6, + "ttft_s_max": 0.09, + "timing_valid": true, + "end_to_end_tps": 659.4195276485798, + "makespan_s": 1.552880915813148, + "steady_tps": 693.4, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 656.0870307437502, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5765669997781515, + "end_to_end_tps": 649.5125168445701, + "timing_valid": true, + "steady_tps": 681.3889181243284, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 666.8343559160052, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.552880915813148, + "end_to_end_tps": 659.4195276485798, + "timing_valid": true, + "steady_tps": 693.4065691010495, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 670.7889683165857, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5437024589627981, + "end_to_end_tps": 663.3402661598517, + "timing_valid": true, + "steady_tps": 697.231299119305, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 291.8, + "per_stream_tps": 291.8, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 285.8762741838799, + "makespan_s": 0.895492292009294, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 291.1196241194596, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8974603326059878, + "end_to_end_tps": 285.24937615531553, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 291.76905205557676, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.895492292009294, + "end_to_end_tps": 285.8762741838799, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 291.855333283309, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8952422910369933, + "end_to_end_tps": 285.9561065903907, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 680.9, + "per_stream_tps": 171.6, + "ttft_s_max": 0.09, + "timing_valid": true, + "end_to_end_tps": 672.8632729392803, + "makespan_s": 1.521854500286281, + "steady_tps": 709.4, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 646.1587627528896, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.6007841248065233, + "end_to_end_tps": 639.6865037150243, + "timing_valid": true, + "steady_tps": 671.5135302935957, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 681.7934959731334, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5196675001643598, + "end_to_end_tps": 673.8316111183856, + "timing_valid": true, + "steady_tps": 710.3947347572583, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 680.872689599641, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.521854500286281, + "end_to_end_tps": 672.8632729392803, + "timing_valid": true, + "steady_tps": 709.4430155143721, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + } + ] + } + ] + }, + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.8273167083971202, + "elapsed_s": 0.9130229582078755, + "engine_prefill_s": 0.815814416, + "full_token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89", + "output_text_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "normalized_body_sha256": "b9bc5e2feab4faa89e3cff58510809a260dfe738c75102107e2ccb2a018c3f3e", + "peak_physical_process_footprint_bytes": 2150369320 + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 3.833395458292216, + "elapsed_s": 3.9251832920126617, + "engine_prefill_s": 3.820862416, + "full_token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "output_text_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "normalized_body_sha256": "e4d6edf5df4aa5f7d6633ccf51897404f88b486f5cc919ed353e40acb5aa8ce1", + "peak_physical_process_footprint_bytes": 2289240152 + }, + { + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 20.915167040657252, + "elapsed_s": 21.001872832886875, + "engine_prefill_s": 20.89837125, + "full_token_sha256": "f35dfbfd86653608bd3aa0fe6672229ecfe9ec17bf45ebcdb91dc7c7e894af4a", + "output_text_sha256": "2acefce2656f36e21405907d799ceb7a85e7557924451ed2c622c79474fa0999", + "normalized_body_sha256": "667df3a5602ea177f4c8014d7c12c748d3df65fa026d4891a8dd87977c8c8fcf", + "peak_physical_process_footprint_bytes": 2844444808 + }, + { + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 59.694949250202626, + "elapsed_s": 59.782952167093754, + "engine_prefill_s": 59.671945541, + "full_token_sha256": "6dc17e16c1bdd265cbfecc43f466db1fb29c9240e7818d912ecc773b568b4856", + "output_text_sha256": "eb30855e72463651ef28f847b1464ebf1651787ab4add8a37e5bd90565054e97", + "normalized_body_sha256": "f82e80f89b846734ed458943f26d45bac00bfdae2fe2c20e0680ef1060e490ca", + "peak_physical_process_footprint_bytes": 3585017968 + } + ] + }, + { + "family": "vl", + "bits": 4, + "engine": "standard", + "decode": [ + { + "label": "vl-4-standard", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 323.99325739582156, + "decode_tps_all": [ + 324.01, + 323.76, + 323.99 + ], + "ttft_s_median": 0.018338708207011223, + "prefill_tps_median": 2890.062, + "itl_mean_ms_median": 3.086, + "itl_p50_ms_median": 3.06, + "itl_p95_ms_median": 3.162, + "round_gap_p50_ms_median": 3.06, + "round_gap_p95_ms_median": 3.162, + "tokens_per_chunk_median": 1.0, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "8f09aba86abc10a48c2aa0da4db383e8a734db540e2ac890f07b5e39578e9502" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "8f09aba86abc10a48c2aa0da4db383e8a734db540e2ac890f07b5e39578e9502" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "8f09aba86abc10a48c2aa0da4db383e8a734db540e2ac890f07b5e39578e9502" + } + ] + }, + { + "label": "vl-4-standard", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 324.8657500934113, + "decode_tps_all": [ + 324.87, + 324.72, + 325.01 + ], + "ttft_s_median": 0.01815433381125331, + "prefill_tps_median": 2974.496, + "itl_mean_ms_median": 3.078, + "itl_p50_ms_median": 3.052, + "itl_p95_ms_median": 3.158, + "round_gap_p50_ms_median": 3.052, + "round_gap_p95_ms_median": 3.158, + "tokens_per_chunk_median": 1.0, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "8c84c68381968d2a7264db9bdcd270d793ac6ecb227c7272d13158931f57cf17" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "8c84c68381968d2a7264db9bdcd270d793ac6ecb227c7272d13158931f57cf17" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "8c84c68381968d2a7264db9bdcd270d793ac6ecb227c7272d13158931f57cf17" + } + ] + } + ], + "concurrent": { + "label": "vl-4-standard", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 324.9, + "per_stream_tps": 324.9, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 318.75760926841656, + "makespan_s": 0.8031180826947093, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 324.3295664010408, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8057073326781392, + "end_to_end_tps": 317.7332383820638, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f09aba86abc10a48c2aa0da4db383e8a734db540e2ac890f07b5e39578e9502" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 324.91514624711414, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8031180826947093, + "end_to_end_tps": 318.75760926841656, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f09aba86abc10a48c2aa0da4db383e8a734db540e2ac890f07b5e39578e9502" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 325.0408262607277, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.80292550008744, + "end_to_end_tps": 318.8340636486463, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f09aba86abc10a48c2aa0da4db383e8a734db540e2ac890f07b5e39578e9502" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 940.3, + "per_stream_tps": 239.0, + "ttft_s_max": 0.09, + "timing_valid": true, + "end_to_end_tps": 915.8246612698283, + "makespan_s": 1.118117957841605, + "steady_tps": 953.7, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 908.6885995999833, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1559775420464575, + "end_to_end_tps": 885.830358077014, + "timing_valid": true, + "steady_tps": 952.2047384302256, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 940.3397447932114, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.118117957841605, + "end_to_end_tps": 915.8246612698283, + "timing_valid": true, + "steady_tps": 953.7361605064481, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 940.3582398547433, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1175956670194864, + "end_to_end_tps": 916.2526575741864, + "timing_valid": true, + "steady_tps": 953.6516786297474, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 324.1, + "per_stream_tps": 324.1, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 318.0112206858499, + "makespan_s": 0.8050030418671668, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 324.123357636115, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8050030418671668, + "end_to_end_tps": 318.0112206858499, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8c84c68381968d2a7264db9bdcd270d793ac6ecb227c7272d13158931f57cf17" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 324.3225195683169, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8048065421171486, + "end_to_end_tps": 318.0888655881929, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8c84c68381968d2a7264db9bdcd270d793ac6ecb227c7272d13158931f57cf17" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 324.03549148642657, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8054475001990795, + "end_to_end_tps": 317.83573719792463, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8c84c68381968d2a7264db9bdcd270d793ac6ecb227c7272d13158931f57cf17" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 940.7, + "per_stream_tps": 238.4, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 916.6032063427284, + "makespan_s": 1.117168250028044, + "steady_tps": 953.1, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 932.3781685537547, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1271923342719674, + "end_to_end_tps": 908.4518842663906, + "timing_valid": true, + "steady_tps": 944.6865073197674, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "fa2d3eed4c2582f3077ad00bb99b4e2db17605f9bcc8dab56f3fa87c696c08e6" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "fa2d3eed4c2582f3077ad00bb99b4e2db17605f9bcc8dab56f3fa87c696c08e6" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "372fd48a132419570d44767f9832f28ab8e663fbd6d31e8a2db438ef0ca86530" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "fa2d3eed4c2582f3077ad00bb99b4e2db17605f9bcc8dab56f3fa87c696c08e6" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 940.7244089287482, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.117168250028044, + "end_to_end_tps": 916.6032063427284, + "timing_valid": true, + "steady_tps": 953.1214704507041, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "fa2d3eed4c2582f3077ad00bb99b4e2db17605f9bcc8dab56f3fa87c696c08e6" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "fa2d3eed4c2582f3077ad00bb99b4e2db17605f9bcc8dab56f3fa87c696c08e6" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "372fd48a132419570d44767f9832f28ab8e663fbd6d31e8a2db438ef0ca86530" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "fa2d3eed4c2582f3077ad00bb99b4e2db17605f9bcc8dab56f3fa87c696c08e6" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 942.3111925057592, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1154986671172082, + "end_to_end_tps": 917.9750995546513, + "timing_valid": true, + "steady_tps": 954.6658833434484, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "fa2d3eed4c2582f3077ad00bb99b4e2db17605f9bcc8dab56f3fa87c696c08e6" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "372fd48a132419570d44767f9832f28ab8e663fbd6d31e8a2db438ef0ca86530" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "fa2d3eed4c2582f3077ad00bb99b4e2db17605f9bcc8dab56f3fa87c696c08e6" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "fa2d3eed4c2582f3077ad00bb99b4e2db17605f9bcc8dab56f3fa87c696c08e6" + } + ] + } + ] + } + ] + }, + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.15338558284565806, + "elapsed_s": 0.2004660409875214, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "normalized_body_sha256": "b9bc5e2feab4faa89e3cff58510809a260dfe738c75102107e2ccb2a018c3f3e", + "peak_physical_process_footprint_bytes": 3175877176 + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.42749424977228045, + "elapsed_s": 0.48020175006240606, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "normalized_body_sha256": "e4d6edf5df4aa5f7d6633ccf51897404f88b486f5cc919ed353e40acb5aa8ce1", + "peak_physical_process_footprint_bytes": 3432254032 + }, + { + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 1.721422334201634, + "elapsed_s": 1.7821548753418028, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "2acefce2656f36e21405907d799ceb7a85e7557924451ed2c622c79474fa0999", + "normalized_body_sha256": "667df3a5602ea177f4c8014d7c12c748d3df65fa026d4891a8dd87977c8c8fcf", + "peak_physical_process_footprint_bytes": 4139338368 + }, + { + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 4.0668870829977095, + "elapsed_s": 4.135986499954015, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "eb30855e72463651ef28f847b1464ebf1651787ab4add8a37e5bd90565054e97", + "normalized_body_sha256": "f82e80f89b846734ed458943f26d45bac00bfdae2fe2c20e0680ef1060e490ca", + "peak_physical_process_footprint_bytes": 5027973880 + } + ] + }, + { + "family": "vl", + "bits": 6, + "engine": "native", + "decode": [ + { + "label": "vl-6-native", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 269.61832352648776, + "decode_tps_all": [ + 269.62, + 269.66, + 268.81 + ], + "ttft_s_median": 0.023640542291104794, + "prefill_tps_median": 2241.911, + "itl_mean_ms_median": 3.709, + "itl_p50_ms_median": 3.787, + "itl_p95_ms_median": 4.243, + "round_gap_p50_ms_median": 3.796, + "round_gap_p95_ms_median": 4.243, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + } + ] + }, + { + "label": "vl-6-native", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 264.5344154729959, + "decode_tps_all": [ + 265.3, + 264.53, + 264.51 + ], + "ttft_s_median": 0.023763916920870543, + "prefill_tps_median": 2272.353, + "itl_mean_ms_median": 3.78, + "itl_p50_ms_median": 3.794, + "itl_p95_ms_median": 4.251, + "round_gap_p50_ms_median": 3.8, + "round_gap_p95_ms_median": 4.251, + "tokens_per_chunk_median": 1.008, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + } + ] + } + ], + "concurrent": { + "label": "vl-6-native", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 269.4, + "per_stream_tps": 269.4, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 263.8627245182019, + "makespan_s": 0.970201457850635, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 269.09506498912566, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9712019581347704, + "end_to_end_tps": 263.5909018260811, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 269.3894248300511, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.970201457850635, + "end_to_end_tps": 263.8627245182019, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 269.7845054848385, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9687430830672383, + "end_to_end_tps": 264.25995134793817, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 616.1, + "per_stream_tps": 157.3, + "ttft_s_max": 0.09, + "timing_valid": true, + "end_to_end_tps": 609.1044306601523, + "makespan_s": 1.6811567088589072, + "steady_tps": 643.6, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 619.3480694531077, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.671424750238657, + "end_to_end_tps": 612.6509732812001, + "timing_valid": true, + "steady_tps": 646.8565244500825, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 612.7717861060737, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.6899075000546873, + "end_to_end_tps": 605.9503256638972, + "timing_valid": true, + "steady_tps": 639.6625911994137, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 616.129391621727, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.6811567088589072, + "end_to_end_tps": 609.1044306601523, + "timing_valid": true, + "steady_tps": 643.6285616329899, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 264.9, + "per_stream_tps": 264.9, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 259.4883780677488, + "makespan_s": 0.986556707881391, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 264.5504931042794, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9876990839838982, + "end_to_end_tps": 259.18825293167265, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 264.859322057916, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.986556707881391, + "end_to_end_tps": 259.4883780677488, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 265.09485640963413, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9857115410268307, + "end_to_end_tps": 259.71086808349725, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 612.6, + "per_stream_tps": 155.2, + "ttft_s_max": 0.1, + "timing_valid": true, + "end_to_end_tps": 605.9453207129101, + "makespan_s": 1.6899214582517743, + "steady_tps": 640.9, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 612.5909665981223, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.6899214582517743, + "end_to_end_tps": 605.9453207129101, + "timing_valid": true, + "steady_tps": 640.9013509631529, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 592.9545084879833, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.745702541898936, + "end_to_end_tps": 586.5833241476041, + "timing_valid": true, + "steady_tps": 619.8145800703737, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 620.0612615509974, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.670743083115667, + "end_to_end_tps": 612.9009363249344, + "timing_valid": true, + "steady_tps": 649.2582133846124, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + } + } + ] + } + ] + } + ] + }, + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.9108745418488979, + "elapsed_s": 1.0000995001755655, + "engine_prefill_s": 0.899594416, + "full_token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89", + "output_text_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "normalized_body_sha256": "b9bc5e2feab4faa89e3cff58510809a260dfe738c75102107e2ccb2a018c3f3e", + "peak_physical_process_footprint_bytes": 2343586120 + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 3.9665978751145303, + "elapsed_s": 4.054472499992698, + "engine_prefill_s": 3.953866833, + "full_token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "output_text_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "normalized_body_sha256": "e4d6edf5df4aa5f7d6633ccf51897404f88b486f5cc919ed353e40acb5aa8ce1", + "peak_physical_process_footprint_bytes": 2482129176 + }, + { + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 21.879985665902495, + "elapsed_s": 21.96919879084453, + "engine_prefill_s": 21.8627035, + "full_token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "output_text_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "normalized_body_sha256": "667df3a5602ea177f4c8014d7c12c748d3df65fa026d4891a8dd87977c8c8fcf", + "peak_physical_process_footprint_bytes": 3037235480 + }, + { + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 62.44881037482992, + "elapsed_s": 62.54085320793092, + "engine_prefill_s": 62.427248292, + "full_token_sha256": "01496c951534bedc261fc706cd6de05a865b636542badf970ce10254c9b17dfd", + "output_text_sha256": "4026f32b09f16475aea9189543a259259e5dea4a9c58e40d969af677ffb1251b", + "normalized_body_sha256": "f82e80f89b846734ed458943f26d45bac00bfdae2fe2c20e0680ef1060e490ca", + "peak_physical_process_footprint_bytes": 3777579312 + } + ] + }, + { + "family": "vl", + "bits": 6, + "engine": "standard", + "decode": [ + { + "label": "vl-6-standard", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 288.7499754512207, + "decode_tps_all": [ + 288.56, + 288.75, + 288.77 + ], + "ttft_s_median": 0.018629542086273432, + "prefill_tps_median": 2844.944, + "itl_mean_ms_median": 3.463, + "itl_p50_ms_median": 3.44, + "itl_p95_ms_median": 3.483, + "round_gap_p50_ms_median": 3.44, + "round_gap_p95_ms_median": 3.483, + "tokens_per_chunk_median": 1.0, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "287181355b709d4fa88ceea7f9df808451e2a85de2f33541cd350619b8fdc4db" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "287181355b709d4fa88ceea7f9df808451e2a85de2f33541cd350619b8fdc4db" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "287181355b709d4fa88ceea7f9df808451e2a85de2f33541cd350619b8fdc4db" + } + ] + }, + { + "label": "vl-6-standard", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 287.93558373666843, + "decode_tps_all": [ + 287.92, + 287.96, + 287.94 + ], + "ttft_s_median": 0.018687416333705187, + "prefill_tps_median": 2889.645, + "itl_mean_ms_median": 3.473, + "itl_p50_ms_median": 3.451, + "itl_p95_ms_median": 3.497, + "round_gap_p50_ms_median": 3.451, + "round_gap_p95_ms_median": 3.497, + "tokens_per_chunk_median": 1.0, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "1ad92991b86c2803dbada765356db0bba6488a52dfb4372aeb8a8dd6d09017af" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "1ad92991b86c2803dbada765356db0bba6488a52dfb4372aeb8a8dd6d09017af" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "1ad92991b86c2803dbada765356db0bba6488a52dfb4372aeb8a8dd6d09017af" + } + ] + } + ], + "concurrent": { + "label": "vl-6-standard", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 284.8, + "per_stream_tps": 284.8, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 279.41145503716433, + "makespan_s": 0.9162115417420864, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 284.8229176492739, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9162115417420864, + "end_to_end_tps": 279.41145503716433, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "287181355b709d4fa88ceea7f9df808451e2a85de2f33541cd350619b8fdc4db" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 284.4766132105267, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9164409171789885, + "end_to_end_tps": 279.341521314899, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "287181355b709d4fa88ceea7f9df808451e2a85de2f33541cd350619b8fdc4db" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 285.0453738224251, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9146781247109175, + "end_to_end_tps": 279.8798758644287, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "287181355b709d4fa88ceea7f9df808451e2a85de2f33541cd350619b8fdc4db" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 921.1, + "per_stream_tps": 234.2, + "ttft_s_max": 0.06, + "timing_valid": true, + "end_to_end_tps": 897.2409079343823, + "makespan_s": 1.1412765411660075, + "steady_tps": 928.5, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 919.5529060559651, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1460797502659261, + "end_to_end_tps": 893.480580005362, + "timing_valid": true, + "steady_tps": 927.9381243667546, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "f31d2a54afa11a9a739ad8a4cfeb2984d3724392e8ebf4b1f41d755a2a4998b3" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "021f28600cc8b366f0b0d6d06b2523d2d476978e7a82bfeb14298f820323b60c" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "021f28600cc8b366f0b0d6d06b2523d2d476978e7a82bfeb14298f820323b60c" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "021f28600cc8b366f0b0d6d06b2523d2d476978e7a82bfeb14298f820323b60c" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 921.1380523510907, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1412765411660075, + "end_to_end_tps": 897.2409079343823, + "timing_valid": true, + "steady_tps": 928.5494141547683, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "021f28600cc8b366f0b0d6d06b2523d2d476978e7a82bfeb14298f820323b60c" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "021f28600cc8b366f0b0d6d06b2523d2d476978e7a82bfeb14298f820323b60c" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "021f28600cc8b366f0b0d6d06b2523d2d476978e7a82bfeb14298f820323b60c" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "f31d2a54afa11a9a739ad8a4cfeb2984d3724392e8ebf4b1f41d755a2a4998b3" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 923.3861812879765, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1386199998669326, + "end_to_end_tps": 899.3342819550614, + "timing_valid": true, + "steady_tps": 930.881924505347, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "f31d2a54afa11a9a739ad8a4cfeb2984d3724392e8ebf4b1f41d755a2a4998b3" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "021f28600cc8b366f0b0d6d06b2523d2d476978e7a82bfeb14298f820323b60c" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "021f28600cc8b366f0b0d6d06b2523d2d476978e7a82bfeb14298f820323b60c" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "021f28600cc8b366f0b0d6d06b2523d2d476978e7a82bfeb14298f820323b60c" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 288.8, + "per_stream_tps": 288.8, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 283.9223821938573, + "makespan_s": 0.9016548749059439, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 289.0246980741583, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9009908330626786, + "end_to_end_tps": 284.1316366447327, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "1ad92991b86c2803dbada765356db0bba6488a52dfb4372aeb8a8dd6d09017af" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 288.8189954950191, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9016548749059439, + "end_to_end_tps": 283.9223821938573, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "1ad92991b86c2803dbada765356db0bba6488a52dfb4372aeb8a8dd6d09017af" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 288.5999485524134, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9023782918229699, + "end_to_end_tps": 283.6947678371484, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "1ad92991b86c2803dbada765356db0bba6488a52dfb4372aeb8a8dd6d09017af" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 921.9, + "per_stream_tps": 234.0, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 898.0719409807541, + "makespan_s": 1.1402204581536353, + "steady_tps": 944.1, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 917.5195980769362, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.145505000371486, + "end_to_end_tps": 893.9288782396567, + "timing_valid": true, + "steady_tps": 939.7449351845229, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9dfa53730911dc2bc4e4aacfa8761dde28da710f897cea892edceff92040d8c8" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9dfa53730911dc2bc4e4aacfa8761dde28da710f897cea892edceff92040d8c8" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "c74d54c61041441ab4b794e9c8af4271969b723bc1eceeb0ea8d00bf2eec1f60" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9dfa53730911dc2bc4e4aacfa8761dde28da710f897cea892edceff92040d8c8" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 923.2887713758215, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1389184170402586, + "end_to_end_tps": 899.0986401476406, + "timing_valid": true, + "steady_tps": 945.2864621866044, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9dfa53730911dc2bc4e4aacfa8761dde28da710f897cea892edceff92040d8c8" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9dfa53730911dc2bc4e4aacfa8761dde28da710f897cea892edceff92040d8c8" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9dfa53730911dc2bc4e4aacfa8761dde28da710f897cea892edceff92040d8c8" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "c74d54c61041441ab4b794e9c8af4271969b723bc1eceeb0ea8d00bf2eec1f60" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 921.9191354761754, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1402204581536353, + "end_to_end_tps": 898.0719409807541, + "timing_valid": true, + "steady_tps": 944.0626208973727, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9dfa53730911dc2bc4e4aacfa8761dde28da710f897cea892edceff92040d8c8" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9dfa53730911dc2bc4e4aacfa8761dde28da710f897cea892edceff92040d8c8" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "c74d54c61041441ab4b794e9c8af4271969b723bc1eceeb0ea8d00bf2eec1f60" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9dfa53730911dc2bc4e4aacfa8761dde28da710f897cea892edceff92040d8c8" + } + ] + } + ] + } + ] + }, + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.16004212526604533, + "elapsed_s": 0.2135536251589656, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "normalized_body_sha256": "b9bc5e2feab4faa89e3cff58510809a260dfe738c75102107e2ccb2a018c3f3e", + "peak_physical_process_footprint_bytes": 3330230984 + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.4499815423041582, + "elapsed_s": 0.5084462920203805, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "normalized_body_sha256": "e4d6edf5df4aa5f7d6633ccf51897404f88b486f5cc919ed353e40acb5aa8ce1", + "peak_physical_process_footprint_bytes": 3578104544 + }, + { + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 1.7762596672400832, + "elapsed_s": 1.8411783752962947, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "cd68ec38f174ad2fc03030a610282a0678fb8c6662ab51a33bf72ca2bc7f3192", + "normalized_body_sha256": "667df3a5602ea177f4c8014d7c12c748d3df65fa026d4891a8dd87977c8c8fcf", + "peak_physical_process_footprint_bytes": 4303932128 + }, + { + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 4.217463042121381, + "elapsed_s": 4.29728762479499, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "4026f32b09f16475aea9189543a259259e5dea4a9c58e40d969af677ffb1251b", + "normalized_body_sha256": "f82e80f89b846734ed458943f26d45bac00bfdae2fe2c20e0680ef1060e490ca", + "peak_physical_process_footprint_bytes": 5212457840 + } + ] + }, + { + "family": "vl", + "bits": 8, + "engine": "native", + "decode": [ + { + "label": "vl-8-native", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 256.7852569225711, + "decode_tps_all": [ + 257.3, + 256.42, + 256.79 + ], + "ttft_s_median": 0.020820292178541422, + "prefill_tps_median": 2545.593, + "itl_mean_ms_median": 3.894, + "itl_p50_ms_median": 3.961, + "itl_p95_ms_median": 4.424, + "round_gap_p50_ms_median": 3.98, + "round_gap_p95_ms_median": 4.424, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + } + ] + }, + { + "label": "vl-8-native", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 256.2428740302236, + "decode_tps_all": [ + 256.24, + 255.47, + 256.5 + ], + "ttft_s_median": 0.02111829211935401, + "prefill_tps_median": 2557.025, + "itl_mean_ms_median": 3.903, + "itl_p50_ms_median": 3.976, + "itl_p95_ms_median": 4.461, + "round_gap_p50_ms_median": 4.001, + "round_gap_p95_ms_median": 4.461, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + } + ] + } + ], + "concurrent": { + "label": "vl-8-native", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 256.1, + "per_stream_tps": 256.1, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 251.86543986964787, + "makespan_s": 1.0164157501421869, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 256.23038628108355, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.016065999865532, + "end_to_end_tps": 251.95213700082428, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 256.1414730196153, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.0164157501421869, + "end_to_end_tps": 251.86543986964787, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 255.54039357041125, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.0187427499331534, + "end_to_end_tps": 251.290131897182, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 623.6, + "per_stream_tps": 159.6, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 617.4444330292528, + "makespan_s": 1.6584488339722157, + "steady_tps": 646.5, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 642.7309429843236, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.6086814999580383, + "end_to_end_tps": 636.5461404427853, + "timing_valid": true, + "steady_tps": 667.1678159145224, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 613.3498451409845, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.685039249714464, + "end_to_end_tps": 607.7009779882104, + "timing_valid": true, + "steady_tps": 635.9930718678236, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 623.6073756036936, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.6584488339722157, + "end_to_end_tps": 617.4444330292528, + "timing_valid": true, + "steady_tps": 646.5248153914514, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 256.1, + "per_stream_tps": 256.1, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 251.7815674417468, + "makespan_s": 1.0167543343268335, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 256.18804001419096, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.0164734586142004, + "end_to_end_tps": 251.8511406574405, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 256.1188768627539, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.0167543343268335, + "end_to_end_tps": 251.7815674417468, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 255.83251094274215, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.0178676657378674, + "end_to_end_tps": 251.50617179142026, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 617.1, + "per_stream_tps": 159.2, + "ttft_s_max": 0.09, + "timing_valid": true, + "end_to_end_tps": 610.8423779205397, + "makespan_s": 1.6763735408894718, + "steady_tps": 640.3, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 616.1487449737187, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.6773182502947748, + "end_to_end_tps": 610.4983355544128, + "timing_valid": true, + "steady_tps": 638.9218686380012, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 617.0910628499275, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.6763735408894718, + "end_to_end_tps": 610.8423779205397, + "timing_valid": true, + "steady_tps": 640.2897178035472, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 678.3030552944775, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5273699997924268, + "end_to_end_tps": 670.433490339056, + "timing_valid": true, + "steady_tps": 706.4627135489969, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + } + } + ] + } + ] + } + ] + }, + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.8097140830941498, + "elapsed_s": 0.9022893747314811, + "engine_prefill_s": 0.798335458, + "full_token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89", + "output_text_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "normalized_body_sha256": "b9bc5e2feab4faa89e3cff58510809a260dfe738c75102107e2ccb2a018c3f3e", + "peak_physical_process_footprint_bytes": 2775910992 + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 3.5564570417627692, + "elapsed_s": 3.6467764168046415, + "engine_prefill_s": 3.544671625, + "full_token_sha256": "8ed4e05be9f053ca054d2c47326a04f0d7fb0c230374bc8bbafaa9618d59ed38", + "output_text_sha256": "b7b834e8f232d6ff122c84f11c241563a5b5bd6ae97dfaabd0e38994edaff19e", + "normalized_body_sha256": "e4d6edf5df4aa5f7d6633ccf51897404f88b486f5cc919ed353e40acb5aa8ce1", + "peak_physical_process_footprint_bytes": 2914667112 + }, + { + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 20.23144816700369, + "elapsed_s": 20.323921124916524, + "engine_prefill_s": 20.214830167, + "full_token_sha256": "6b68989060076239ed31666b235e266a2e83536962209d719c5e58346bfe6251", + "output_text_sha256": "363f5de76e594bc939dc4dfea2b5c71c087224382a91db7f7aef7ef650b86791", + "normalized_body_sha256": "667df3a5602ea177f4c8014d7c12c748d3df65fa026d4891a8dd87977c8c8fcf", + "peak_physical_process_footprint_bytes": 3469789752 + }, + { + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 59.19886158313602, + "elapsed_s": 59.29409558326006, + "engine_prefill_s": 59.175482042, + "full_token_sha256": "7240d3de4d84db461fc2e1f43a7b9f8a79a876eb933afb924e661ad8d8268c3b", + "output_text_sha256": "b3fc28d243e9bb51d0db0f4729560c1a1382e069e5cee04f555a3eb4233fc207", + "normalized_body_sha256": "f82e80f89b846734ed458943f26d45bac00bfdae2fe2c20e0680ef1060e490ca", + "peak_physical_process_footprint_bytes": 4210182784 + } + ] + }, + { + "family": "vl", + "bits": 8, + "engine": "standard", + "decode": [ + { + "label": "vl-8-standard", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 273.7432577926277, + "decode_tps_all": [ + 273.69, + 273.9, + 273.74 + ], + "ttft_s_median": 0.018855708185583353, + "prefill_tps_median": 2810.82, + "itl_mean_ms_median": 3.653, + "itl_p50_ms_median": 3.628, + "itl_p95_ms_median": 3.668, + "round_gap_p50_ms_median": 3.628, + "round_gap_p95_ms_median": 3.668, + "tokens_per_chunk_median": 1.0, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + } + ] + }, + { + "label": "vl-8-standard", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 273.8349864799858, + "decode_tps_all": [ + 273.83, + 273.88, + 273.2 + ], + "ttft_s_median": 0.018880541902035475, + "prefill_tps_median": 2860.087, + "itl_mean_ms_median": 3.652, + "itl_p50_ms_median": 3.627, + "itl_p95_ms_median": 3.676, + "round_gap_p50_ms_median": 3.627, + "round_gap_p95_ms_median": 3.676, + "tokens_per_chunk_median": 1.0, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "aa1580092ce8b46dc61bd425d5697c7031f2e7e503428df48248be6c514babdf" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "aa1580092ce8b46dc61bd425d5697c7031f2e7e503428df48248be6c514babdf" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "output_sha256": "aa1580092ce8b46dc61bd425d5697c7031f2e7e503428df48248be6c514babdf" + } + ] + } + ], + "concurrent": { + "label": "vl-8-standard", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 268.5, + "per_stream_tps": 268.5, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 263.4734461793993, + "makespan_s": 0.9716349169611931, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 269.21597169601114, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9685165416449308, + "end_to_end_tps": 264.3217632248273, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 267.9250973023736, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9727073749527335, + "end_to_end_tps": 263.1829536734413, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 268.4871851890699, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9716349169611931, + "end_to_end_tps": 263.4734461793993, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 914.9, + "per_stream_tps": 232.4, + "ttft_s_max": 0.06, + "timing_valid": true, + "end_to_end_tps": 890.8144112932522, + "makespan_s": 1.1495099170133471, + "steady_tps": 932.8, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 911.5416250963324, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1637356672436, + "end_to_end_tps": 879.9249080553019, + "timing_valid": true, + "steady_tps": 929.3230014401878, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "fdcda6de66e832d2777852e2cf845291c30e025351d57faf15bd77b02e63bca9" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "fdcda6de66e832d2777852e2cf845291c30e025351d57faf15bd77b02e63bca9" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 914.9883844275644, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1495099170133471, + "end_to_end_tps": 890.8144112932522, + "timing_valid": true, + "steady_tps": 933.218201754052, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "ed6bc26bfcf78fa013de555863f7f44cb91c7b9f3e88ad9f622637ce5936f724" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "ed6bc26bfcf78fa013de555863f7f44cb91c7b9f3e88ad9f622637ce5936f724" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "6dcf1fea36dde63b16b8979e296a35d61fa406958b2411267275674e34654602" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "ed6bc26bfcf78fa013de555863f7f44cb91c7b9f3e88ad9f622637ce5936f724" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 914.9261111186196, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.149200583808124, + "end_to_end_tps": 891.0541940439632, + "timing_valid": true, + "steady_tps": 932.8140855476839, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "ed6bc26bfcf78fa013de555863f7f44cb91c7b9f3e88ad9f622637ce5936f724" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "ed6bc26bfcf78fa013de555863f7f44cb91c7b9f3e88ad9f622637ce5936f724" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "6dcf1fea36dde63b16b8979e296a35d61fa406958b2411267275674e34654602" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "ed6bc26bfcf78fa013de555863f7f44cb91c7b9f3e88ad9f622637ce5936f724" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 269.5, + "per_stream_tps": 269.5, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 265.1153049319167, + "makespan_s": 0.9656175831332803, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 273.1968207214605, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9526754580438137, + "end_to_end_tps": 268.71690441743965, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "aa1580092ce8b46dc61bd425d5697c7031f2e7e503428df48248be6c514babdf" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 269.52063751225234, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9656175831332803, + "end_to_end_tps": 265.1153049319167, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "aa1580092ce8b46dc61bd425d5697c7031f2e7e503428df48248be6c514babdf" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 268.5585358503949, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9706688337028027, + "end_to_end_tps": 263.7356749401738, + "timing_valid": true, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "aa1580092ce8b46dc61bd425d5697c7031f2e7e503428df48248be6c514babdf" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 914.4, + "per_stream_tps": 232.2, + "ttft_s_max": 0.06, + "timing_valid": true, + "end_to_end_tps": 890.2836929479546, + "makespan_s": 1.150195166002959, + "steady_tps": 932.8, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 910.0160310445684, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1585314157418907, + "end_to_end_tps": 883.8776282508139, + "timing_valid": true, + "steady_tps": 929.4193703417069, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "422108b9935d8eda3ede2e626504de1a6145fade72ec58008187e6ddd4f2d9b3" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "422108b9935d8eda3ede2e626504de1a6145fade72ec58008187e6ddd4f2d9b3" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "422108b9935d8eda3ede2e626504de1a6145fade72ec58008187e6ddd4f2d9b3" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "92f645b37b7674a6f5dee7f07ec3b8681778d25d244c6a0330e72690be832634" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 914.4264900441698, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.150195166002959, + "end_to_end_tps": 890.2836929479546, + "timing_valid": true, + "steady_tps": 932.7517976583824, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "422108b9935d8eda3ede2e626504de1a6145fade72ec58008187e6ddd4f2d9b3" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "422108b9935d8eda3ede2e626504de1a6145fade72ec58008187e6ddd4f2d9b3" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "92f645b37b7674a6f5dee7f07ec3b8681778d25d244c6a0330e72690be832634" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "422108b9935d8eda3ede2e626504de1a6145fade72ec58008187e6ddd4f2d9b3" + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 916.0569626953626, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1483942498452961, + "end_to_end_tps": 891.6798391649438, + "timing_valid": true, + "steady_tps": 934.421518056832, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "422108b9935d8eda3ede2e626504de1a6145fade72ec58008187e6ddd4f2d9b3" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "422108b9935d8eda3ede2e626504de1a6145fade72ec58008187e6ddd4f2d9b3" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "422108b9935d8eda3ede2e626504de1a6145fade72ec58008187e6ddd4f2d9b3" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "92f645b37b7674a6f5dee7f07ec3b8681778d25d244c6a0330e72690be832634" + } + ] + } + ] + } + ] + }, + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.1605673748999834, + "elapsed_s": 0.2167126671411097, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "normalized_body_sha256": "b9bc5e2feab4faa89e3cff58510809a260dfe738c75102107e2ccb2a018c3f3e", + "peak_physical_process_footprint_bytes": 3761376352 + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 0.44067829102277756, + "elapsed_s": 0.5023705409839749, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "b7b834e8f232d6ff122c84f11c241563a5b5bd6ae97dfaabd0e38994edaff19e", + "normalized_body_sha256": "e4d6edf5df4aa5f7d6633ccf51897404f88b486f5cc919ed353e40acb5aa8ce1", + "peak_physical_process_footprint_bytes": 4019571664 + }, + { + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 1.7595829591155052, + "elapsed_s": 1.8293205839581788, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "cd68ec38f174ad2fc03030a610282a0678fb8c6662ab51a33bf72ca2bc7f3192", + "normalized_body_sha256": "667df3a5602ea177f4c8014d7c12c748d3df65fa026d4891a8dd87977c8c8fcf", + "peak_physical_process_footprint_bytes": 4726967512 + }, + { + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "complete": true, + "client_ttft_s": 4.19275641720742, + "elapsed_s": 4.273371207993478, + "engine_prefill_s": null, + "full_token_sha256": null, + "output_text_sha256": "ac0fb26c45837a298e9195e00cbefd1205ae81470b3b3e54b3555dd440946ea7", + "normalized_body_sha256": "f82e80f89b846734ed458943f26d45bac00bfdae2fe2c20e0680ef1060e490ca", + "peak_physical_process_footprint_bytes": 5640195192 + } + ] + } + ], + "counts": { + "measured_decode_requests": 72, + "measured_concurrent_requests": 360, + "cold_requests": 48, + "native_alone_equal": 180, + "native_serial_equal": 24 + }, + "decode_prompts": [ + { + "name": "long-code-chat", + "kind": "chat", + "prompt": "Write a complete Python implementation of a red-black tree with insertion, deletion, searching, rotations, and extensive unit tests. Include detailed explanations after each method. Continue until the implementation and every test are complete." + }, + { + "name": "long-prose-chat", + "kind": "chat", + "prompt": "Write a detailed 2000-word tutorial explaining how matrix multiplication runs on a GPU, including memory layout, tiling, shared memory, synchronization, numerical precision, and several worked numerical examples. Explain every step thoroughly." + } + ], + "zig": "0.17.0", + "benchmark_sources_sha256": { + "bench_openai.py": "d5af4fbb756cdfb573546c0e0017652a4bb8f509d1251af683be7a3cbdd426f6", + "bench_concurrent.py": "0e72e26769f082459329fc068c25f6f72a9155e422962ea54cb5d1226227752d", + "bench_evidence.py": "54c28474f0804dbd120e3fb5859be0d1667ef21ca502880d0bed098ec9bb42ff", + "bench_metrics.py": "26f4c3ecbb03df4f12e24d18840fea989d9ef8c63117ca11eb944ef670458a27" + }, + "long_prompt_wrapper_sha256": "48c0709f4c40734a60f0d22fa73adee69f68e40ecc18f43a574447af82fb252e" +} diff --git a/docs/receipts/lfm3/attention-range-text.json b/docs/receipts/lfm3/attention-range-text.json new file mode 100644 index 0000000000..48dd95336d --- /dev/null +++ b/docs/receipts/lfm3/attention-range-text.json @@ -0,0 +1,41 @@ +{ + "scope": "Text-slice applied artifact CPU compilation and source review; no text-binary device rerun or final performance acceptance claimed", + "commit": "4199067fa66ce858ba060b9c7e4e7fae8c65c7f7", + "build": { + "command": "zig build native tf-kernel-contract test-lfm3-host -Dcpu=apple_m1 --summary all", + "actual_exit": 0, + "log": "/private/tmp/lfm3-range-text-build.log" + }, + "format_check": "passed", + "lean": { + "baseline_problems": 428, + "current_problems": 428, + "new_problems": 0 + }, + "text_registry_has_new_lfm3_vision_imports": false, + "source_sha256": { + "zig/kernels/metal/lfm3/attention.metal": "6dc51acbb45e2e65028e72fed72d318629eb7b382e773c5837db9baa8de5d3b9", + "zig/src/families/lfm3/forward.zig": "618e9ecf0e96f4709c0ad543585ec58d06f14501cabdc2b9caeba9ab1d73e7b6", + "zig/src/families/lfm3/state.zig": "1161aea9174fd6a6dbecc09bfd26565459c9d7b8d1674ca06c7c78c1b0265517", + "zig/tests/contract/metal_lfm3.zig": "60abd8dd905a669c13c565241a5537ce9164bf3f9ab2400c70e666af8786e18d", + "zig/tests/contract/metal_lfm3_segments.zig": "434e8215ec6d4459f7b377100bf099f81ccee85a768bb3b6981c6c16ab9b4d29", + "zig/tests/contract/metal_lfm3_attention_sums.zig": "146e77b310349ddee5dc87bfce3d6b05950ce1412c96e1b3cf46e2b247116665", + "zig/tests/contract/metal_lfm3_alpha.zig": "f9df1bdc9d49a8a1847579f79898694ee5b083ec4ff9ac5d1460ca8b2f33d287", + "zig/tests/contract/metal_main.zig": "0c3a9bacccebcd14d6a80227c225c45afe8241ad641dcd6e459dc7d26dbe3a53", + "zig/tests/contract/metal_limits.zig": "5c094102286e9936f96c822277e35e8b065a18f967302d6808420c9cb08d271c", + "zig/src/core/contract/lfm3_records.zig": "b26c6ebf598b95927ccb4fb09d4ad23e6416cc21de5c3c2af9f56b8b793f803f" + }, + "artifact_sha256": { + "zig-out/native/bin/tensorfold-native": "623acac2dd42b51af61bcd9da242a6ec15af55deafdafd46cf4e54054329544f", + "zig-out/bin/tf-kernel-contract": "7216e3e315b252ff6f473d47203257da121997044dfbf3ba0036b55a2153d746" + }, + "guarded_combined_main_source_qualification": { + "state_passed": 408, + "serving_passed": 303, + "public_exact_logit_bytes": 394752000, + "registered_kernels_passed": 70, + "limit_problems": 0, + "receipt": "attention-range.json" + }, + "performance_accepted": false +} diff --git a/docs/receipts/lfm3/attention-range.json b/docs/receipts/lfm3/attention-range.json new file mode 100644 index 0000000000..ca10afc872 --- /dev/null +++ b/docs/receipts/lfm3/attention-range.json @@ -0,0 +1,136 @@ +{ + "scope": "Applied public source matches all ten guarded private candidate source hashes; final applied artifact build pending. No final performance acceptance.", + "complete": { + "actual_exit": 0, + "models": 6, + "state_passed": 408, + "serving_passed": 303, + "all_current_public_logits_exact": true, + "logit_bytes": 394752000 + }, + "models": [ + { + "model": "LFM3-1.7B-MLX-4bit", + "state_passed": 68, + "state_failed": 0, + "serving_counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "logit_bytes": 65792000, + "logits_current_public_exact": true, + "logits_sha256": "f1dbd42c2556fcb05031a00b209c61fadee45e74ee1ff94628d4802479fcb518" + }, + { + "model": "LFM3-1.7B-MLX-6bit", + "state_passed": 68, + "state_failed": 0, + "serving_counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "logit_bytes": 65792000, + "logits_current_public_exact": true, + "logits_sha256": "08bf833a94fa0403bc65c70677d20372b5ab0616fb0a58d26119c486069ef59c" + }, + { + "model": "LFM3-1.7B-MLX-8bit", + "state_passed": 68, + "state_failed": 0, + "serving_counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "logit_bytes": 65792000, + "logits_current_public_exact": true, + "logits_sha256": "1345bd80acaa16fe08a00a14d839e3e37cf9d0d8cd7e98c01b68cd1b8d2f51d4" + }, + { + "model": "LFM3-VL-2B-MLX-4bit", + "state_passed": 68, + "state_failed": 0, + "serving_counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "logit_bytes": 65792000, + "logits_current_public_exact": true, + "logits_sha256": "a4dde0a00ee9acb294c2b5a791e7e73cd05c5d6904ffe171e439ad1b01e25c5d" + }, + { + "model": "LFM3-VL-2B-MLX-6bit", + "state_passed": 68, + "state_failed": 0, + "serving_counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "logit_bytes": 65792000, + "logits_current_public_exact": true, + "logits_sha256": "93ea83c90c5623f4e17502a526e2358f9e1ba3f2d03999295745257dc95bbca6" + }, + { + "model": "LFM3-VL-2B-MLX-8bit", + "state_passed": 68, + "state_failed": 0, + "serving_counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "logit_bytes": 65792000, + "logits_current_public_exact": true, + "logits_sha256": "5a013f3eed97f143429f09a81a829b07dc32360734cd54855e18e514160e0a48" + } + ], + "registered_contracts": { + "passed": 70, + "failed": 0, + "known_findings": 0, + "controls_red": 2, + "controls_missed": 0, + "limit_problems": 0 + }, + "postnorm_control_stage_exit": 0, + "source_and_binary_sha256": { + "/private/tmp/lfm3-combined-qualify.py": "9d9a5f942609de71f0494ce5b23ef0f5bd8f80b65aa9732ff0d5ea601a183a76", + "/private/tmp/lfm3-combined-manifest.json": "2c86619e73c86e6d21a99f1276102294c6c538005af393af5ea4bd6178b61ddc", + "/private/tmp/lfm3-combined-promotion.patch": "58df0a67e783c6d3af89fe18fb878083c9f1055ed09a692340b13612eaf0d9f2", + "/private/tmp/lfm3-postnorm-branchless-combined/zig-out/native/bin/tensorfold-native": "ad72d7ec17b681ede54676ba75f7d4d1b39282aa0dae7cf27d8b201660fce489" + }, + "event_log": "events.json", + "manifest": "manifest.json", + "public_changes_applied": true, + "performance_accepted": false, + "applied_public_source_matches_qualified_private": true, + "committed_or_pushed": false, + "format_check": "passed", + "lean": { + "baseline_problems": 428, + "current_problems": 428, + "new_problems": 0 + }, + "build": { + "status": "passed", + "log": "/private/tmp/lfm3-applied-build.log", + "command": "zig build native tf-kernel-contract tf-lfm3-forward tf-lfm3-exact test-lfm3-host test-lfm3-image-host -Dcpu=apple_m1 --summary all", + "actual_exit": 0 + }, + "performance_scope": { + "acceptance": false, + "isolated_branchless_local_M4_vs_pre_postnorm_baseline_throughput_percent": -5.3498, + "combined_vs_current_public_and_standard": "pending guarded matched local M4 benchmark", + "M5_performance_claim": false + }, + "applied_public_artifact_sha256": { + "zig-out/native/bin/tensorfold-native": "3d4a3daeda697420bfbba6d81972896c2bf9e31efaf6f12321ee8487a299354b", + "zig-out/bin/tf-kernel-contract": "2786719bfe0314754349463721b8e1faa7cd06f6f2efa142db19acacd83d54e6", + "zig-out/bin/tf-lfm3-forward": "243da3eb79553e7937c36959db13df84baa9d7fcc3e4724d164a708f2fd1eacc", + "zig-out/bin/tf-lfm3-exact": "e078bbbe23f5a36ac7813b9cef1dab6b9ecfd404d39a99ec21969e3ee5cd55ad" + } +} diff --git a/docs/receipts/lfm3/binned-nonfinite.json b/docs/receipts/lfm3/binned-nonfinite.json new file mode 100644 index 0000000000..0a19f12dae --- /dev/null +++ b/docs/receipts/lfm3/binned-nonfinite.json @@ -0,0 +1,145 @@ +{ + "status": "verified; binned candidate remains disabled by default", + "regression": { + "inputs": "BF16 X=+1 [1,64], codes=1, scale=+1, bias=+Inf, group64, bits4", + "before": "NaN (0x7fc0), where the independent factored FP64 expression is +Inf", + "cause": "FMA product residual inf-inf contaminated the low component of an infinite product.", + "fix": "Keep the original computed compensated product when both components are finite. On nonfinite/overflow results only, normalize the input pair and recompute, returning nonfinite results with a zero low component.", + "before_public_shader_sha256": "8119aa792197ebf3c9b12f767fe0dfe12482189340be8e75ae93f7910ac944ca", + "probe_binary_sha256": "18a8d11c077ae817f560d7ed31ea0045c8a1b2c058612d793feb60b627025588", + "before_probe_shader_hash": "not retained; original arithmetic is re-executed by the registered nonfinite negative control", + "finite_domain_scope": "The operation order is unchanged for original compensated products whose two components are finite. Finite operands that overflow can intentionally change; this is not a universal all-finite-input equivalence claim." + }, + "native_nonfinite_probe": { + "shader_sha256": "3fd70fb76ff2ea9ec12b55f3cccb7a2c8ddeeb1e8010f244a7910b101e1d463c", + "cases_per_host": 30, + "raw_candidate_corrected_canonical_pairs_per_host": 960, + "finite_outputs_per_host": 96, + "nonfinite_outputs_per_host": 864, + "axes": [ + "input", + "scale", + "bias" + ], + "planted_values": [ + "NaN", + "+Inf", + "-Inf" + ], + "bits": [ + 4, + 6, + 8 + ], + "group": 64, + "hosts": { + "M4 Pro": { + "raw_bytes_equal": true, + "signed_inf_nan_classes_match": true, + "nonfinite_outputs_certified": 0 + }, + "M5 Ultra": { + "raw_bytes_equal": true, + "signed_inf_nan_classes_match": true, + "nonfinite_outputs_certified": 0 + } + } + }, + "kernel_contract": { + "fixture": "Existing nonfinite input rows plus a positive finite last row, NaN and both infinities in scales/biases, and q6/q8 nibble/bin cancellation leaving (high=0,low=2).", + "new_control": "TF_BP_BAD_NONFINITE restores the original product-residual bug and must fail gate1.", + "M4 Pro": { + "kernels_pass": 6, + "kernels_fail": 0, + "controls_red_as_planted": 3, + "controls_missed": 0, + "limit_problems": 0 + }, + "M5 Ultra": { + "kernels_pass": 6, + "kernels_fail": 0, + "controls_red_as_planted": 3, + "controls_missed": 0, + "limit_problems": 0 + } + }, + "finite_checkpoint_proof": { + "host": "M5 Ultra", + "cases": 132, + "bytes": 123814080, + "all_canonical_bytes_equal": true, + "prompt_decode_pairs": 66, + "all_prompt_decode_first_row_bytes_equal": true, + "comparison": "All132 captures match the frozen FP64-exact canonical captures in binned-projection.json; includes six unchanged n16 fallbacks.", + "binary_sha256": "18d7dbf6148dc0541dc69422b0a4026fe2aa325482a1941095ffcda5da2eb465", + "capture_manifest_sha256": "60c0243fd00107f3e16c73efb3022d02ad19d50022d3fb52414597687f525bdf" + }, + "artifacts": { + "project": "18d7dbf6148dc0541dc69422b0a4026fe2aa325482a1941095ffcda5da2eb465", + "contract": "09506e643820fecac7be980dff611bb96f010e81b7e5239238fbc76527665b87" + }, + "source_sha256": { + "zig/kernels/metal/core/binned_projection.metal": "3029d1cee80ad6897d0a5dd3e788291ad9bb5f74d1fe1fb689c04368569d618f", + "zig/tests/contract/metal_binned_projection.zig": "100d93e632d73d5ce73221646e4ce2d1be42f06ac3aa9b0862a667c3262e31b6", + "zig/tests/contract/metal_main.zig": "b6e42170977de56d6c12c68c994b7b7fe1fd0f2af7f5865a94d3274116e548db", + "zig/tests/contract/metal_limits.zig": "a898d53688271ca6da39daac167e582d7237c9e24d8ba849b211e1b3a9b3a864", + "zig/src/core/contract/lfm3_records.zig": "9ba50b5b5e4d0ffc9834e2c83c6b4c11872e57758b6372402ee293a0d41fd34f" + }, + "scope": "No production candidate activation or performance acceptance is asserted by this receipt.", + "reproduce_kernel_contract": [ + "zig build tf-kernel-contract tf-lfm3-project", + "zig-out/bin/tf-kernel-contract --only 'LFM3 binned'" + ], + "tested_source_commit": "43e5f1a90", + "comment_only_successor_commit": "8955faee0", + "comment_only_successor_scope": "Comments only; tested mathematical operations and line positions are unchanged. The source hashes below identify the actual pre-comment binaries, not the successor.", + "low_component_regression": { + "before_shader_sha256": "c839dbf5d8945f907b516de9f8ca0cb2e7d47b8acc4d63c349dc22ec863e1a3c", + "operator_fixture": { + "bits": [ + 6, + 8 + ], + "input_first_group": [ + "2^26", + "-2^22", + "2", + "remaining entries zero" + ], + "codes_first_group": [ + 1, + 16, + 1 + ], + "remaining_codes": "zero", + "compensated_qx": [ + "high=0", + "low=2" + ], + "scales": [ + "+Inf", + "-Inf", + "NaN" + ], + "bias_inf_columns": "Retained, with a positive input sum; original nonfinite input rows also remain." + }, + "before_native_helper_cases": 4, + "before_native_helper_result": "(0,+/-2)*+Inf produced NaN; finite BF16-maximum products produced signed infinity with a NaN low component.", + "after_native_helper_cases_per_host": 7, + "after_checks": [ + "signed low-only products with +/-Inf", + "low-component overflow with finite BF16 maximum", + "zero represented sum (1,-1) times Inf gives NaN", + "low-component overflow with full finite FP32 maximum" + ], + "after_result": "Six infinite products match raw pair/BF16 bytes with zero low components; the zero-sum case matches the NaN class and a zero low component.", + "M4 Pro": "pass", + "M5 Ultra": "pass", + "probe_program_sha256": "763320e00066edcd336a17a103c666d71ad84db90c2a7c349410a4f58b90e9c1", + "before_probe_shader_sha256": "a474c1dec0889494a33b1bca611d02947fd170caaa49248e321890fd81601e9f", + "after_probe_shader_sha256": "c8e8a0dbf3cf65fd3d64de428690e21a7e8ffa16eb823ce6f9df715a225d421e", + "inputs_sha256": "a92318e04f453f7278b534d246aa528935351e4c4165622a936b4c5fba106057", + "expected_sha256": "d5be7739c64d768ca08422016ae6977a0499aafd7794bc054aa54b6729285d09", + "before_native_helper_host": "M4 Pro" + } +} diff --git a/docs/receipts/lfm3/binned-projection.json b/docs/receipts/lfm3/binned-projection.json new file mode 100644 index 0000000000..618fc9dda2 --- /dev/null +++ b/docs/receipts/lfm3/binned-projection.json @@ -0,0 +1,5540 @@ +{ + "status": "explicit qualification candidate; disabled during ordinary model loading", + "source_commit": "75478ed17", + "shader_sha256": "605188026ccfe9f2e60bd01ee425a954367272c21a0cf6d2e43b7af5ea99261d", + "native_project_tool_sha256": "6c1ed257a51327e0f7d3ae8806fcff488ba07198376432508f4e8d38a5ed1609", + "contract_runner_sha256": "e35dfdacaeba5b35811e60f5c4fe11f951d3025752f746beaae2cb46c761ae14", + "original_row_project_tool_sha256": "244137da5e7afccad8fc6fb97598d53aec96cf2d097d8bfbdf296310ba013428", + "native_platform": { + "gpu": "M5 Ultra", + "ram_gib": 256, + "os": "macOS 27.0 (26A428)", + "zig": "0.17.0", + "path": "nax.h tensor-unit fragments selected by Device.tensorUnits()" + }, + "comparison_platform": { + "gpu": "M4 Pro", + "ram_gb": 48, + "os": "macOS 26.5.1", + "mlx": "0.32.4", + "scope": "Retained original-native and MLX projection captures use identical input rows on M4; this is not a same-device M5 timing comparison." + }, + "reference": { + "arithmetic": "Independent CPU FP64 dequantized checkpoint products", + "rounding": "Direct binary64-to-BF16 round-to-nearest-even", + "scope": "Projection fixtures only; not a whole-model fidelity acceptance policy." + }, + "integer_bin_argument": { + "group64_max_exact_integer_sum": 15667200, + "group32_max_exact_integer_sum": 15667200, + "fp32_exact_integer_limit": 16777216, + "group64_exponent_span": 6, + "group32_exponent_span": 7, + "code_parts": "q = low_nibble + 16 * high_nibble; four-bit codes need only the low part", + "scope": "Finite normal aligned products without overflow/underflow; no universal exactness claim for extreme BF16 subnormals or arbitrary inputs." + }, + "summary": { + "cells": 66, + "prompt_decode_equal_cells": 66, + "outputs": 61907040, + "correctly_rounded": 61907040, + "columns_worse_than_original": 0, + "columns_worse_than_mlx": 0, + "native_hardware": "M5 Ultra", + "comparison_hardware": "M4 Pro", + "binary_sha256": "6c1ed257a51327e0f7d3ae8806fcff488ba07198376432508f4e8d38a5ed1609", + "model_acceptance_threshold": null + }, + "contracts": { + "m4": { + "hardware": "M4 Pro", + "exit": 0, + "summary": "SUMMARY 6 kernels: 6 pass gates 1-4, 0 fail, 0 known findings; 2 controls: 2 red as planted, 0 missed; 0 limit problems; gates 9716 ms, limits 5147 ms" + }, + "m5": { + "hardware": "M5 Ultra", + "exit": 0, + "summary": "SUMMARY 6 kernels: 6 pass gates 1-4, 0 fail, 0 known findings; 2 controls: 2 red as planted, 0 missed; 0 limit problems; gates 8967 ms, limits 918 ms" + } + }, + "fixture_tokens": [ + 89509, + 3420, + 278, + 11622, + 355, + 6542, + 20, + 1256, + 7986, + 1030, + 278, + 3599, + 3306, + 462, + 49422, + 22, + 27407, + 267, + 3983, + 13816, + 1492, + 16526, + 3452, + 22, + 42847, + 3420, + 278, + 11622, + 355, + 6542, + 20, + 1256, + 7986, + 1030, + 278, + 3599, + 3306, + 462, + 49422, + 22, + 27407, + 267, + 3983, + 13816, + 1492, + 16526, + 3452, + 22, + 42847, + 3420, + 278, + 11622, + 355, + 6542, + 20, + 1256, + 7986, + 1030, + 278, + 3599, + 3306, + 462, + 49422, + 22 + ], + "cases": [ + { + "name": "1.7B-4-conv_in", + "class": "conv_in", + "shape": [ + 64, + 2048, + 6144 + ], + "input_sha256": "c3c833599efae0b6f0abeeab1c674d0f88fd3828c89f7e3453505feb1abfa34e", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.00043982049783110983, + "max_abs": 0.007671507657505572, + "worst_column_rms": 0.002848056337488529, + "worst_column": 3716, + "correctly_rounded": 393216, + "outputs": 393216 + }, + "original_native": { + "rms": 0.00043982049813563945, + "max_abs": 0.007671507657505572, + "worst_column_rms": 0.002848056337488529, + "worst_column": 3716, + "correctly_rounded": 393170, + "outputs": 393216 + }, + "mlx": { + "rms": 0.0005148634877763631, + "max_abs": 0.007671507657505572, + "worst_column_rms": 0.002895652676547294, + "worst_column": 3716, + "correctly_rounded": 239774, + "outputs": 393216 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "764c618b648c35902ec16e7f8578687e9b514a4c06ab5e7191c122ce8c704ffe" + }, + { + "mode": "decode", + "native": { + "rms": 0.00048554828079777217, + "max_abs": 0.0037449413212016225, + "worst_column_rms": 0.0037449413212016225, + "worst_column": 1969, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "original_native": { + "rms": 0.00048554828079777217, + "max_abs": 0.0037449413212016225, + "worst_column_rms": 0.0037449413212016225, + "worst_column": 1969, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "mlx": { + "rms": 0.00048554828079777217, + "max_abs": 0.0037449413212016225, + "worst_column_rms": 0.0037449413212016225, + "worst_column": 1969, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "fa0a8412931265b2da7c5083d58a2bb9f9a776a65f8476ee623a7293dc07a1d2" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-4bit", + "module": "language_model.model.layers.0.conv.in_proj", + "input_stage": "layer_00_operator_norm", + "input_file": "1.7B-4-conv_in.bf16", + "one_row_file": "1.7B-4-conv_in-one.bf16", + "model_directory": "LFM3-1.7B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 6144 + }, + { + "name": "1.7B-4-conv_out", + "class": "conv_out", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "a0044e29906efde3689ad19b91868257d8487c11655ac8e83d1e5e73bd3e163e", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 1.602854875446973e-05, + "max_abs": 0.0009396018046068377, + "worst_column_rms": 0.00032164384276088477, + "worst_column": 1688, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 1.602854875697783e-05, + "max_abs": 0.0009396018046068377, + "worst_column_rms": 0.00032164384276088477, + "worst_column": 1688, + "correctly_rounded": 131057, + "outputs": 131072 + }, + "mlx": { + "rms": 1.7543640518602656e-05, + "max_abs": 0.0012221553097768734, + "worst_column_rms": 0.00032858266117196354, + "worst_column": 1688, + "correctly_rounded": 94899, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "72ccfcb68f3a6e49a45a3c61501b886efdf56617095f4ea39c6f8fa6dacf812c" + }, + { + "mode": "decode", + "native": { + "rms": 2.696892560157749e-06, + "max_abs": 5.5160145456811804e-05, + "worst_column_rms": 5.5160145456811804e-05, + "worst_column": 1700, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 2.696892560157749e-06, + "max_abs": 5.5160145456811804e-05, + "worst_column_rms": 5.5160145456811804e-05, + "worst_column": 1700, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "mlx": { + "rms": 2.6968925601881646e-06, + "max_abs": 5.5160145456811804e-05, + "worst_column_rms": 5.5160145456811804e-05, + "worst_column": 1700, + "correctly_rounded": 2047, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "571e0e60f83da426e1f36973ce381c6c07dc0250805f58875cb4c6ce3b16f635" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-4bit", + "module": "language_model.model.layers.0.conv.out_proj", + "input_stage": "layer_00_conv_gated", + "input_file": "1.7B-4-conv_out.bf16", + "one_row_file": "1.7B-4-conv_out-one.bf16", + "model_directory": "LFM3-1.7B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 2048 + }, + { + "name": "1.7B-4-ffn_gate", + "class": "ffn_gate", + "shape": [ + 64, + 2048, + 7680 + ], + "input_sha256": "fa6b92bb7fd858cfc78f98ba720ef3e67984628ffa1c57c7b71834e8a483cfa0", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.000534360851018576, + "max_abs": 0.007667379919439554, + "worst_column_rms": 0.004595861546563877, + "worst_column": 445, + "correctly_rounded": 491520, + "outputs": 491520 + }, + "original_native": { + "rms": 0.0005343608514000784, + "max_abs": 0.007667379919439554, + "worst_column_rms": 0.004595861546563877, + "worst_column": 445, + "correctly_rounded": 491484, + "outputs": 491520 + }, + "mlx": { + "rms": 0.0006128478254610084, + "max_abs": 0.008627567905932665, + "worst_column_rms": 0.004712969625451817, + "worst_column": 445, + "correctly_rounded": 341320, + "outputs": 491520 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "c2f30477ae8dc9605d0624cd0e488921a3d628f5e22756d0d699ceea360166a3" + }, + { + "mode": "decode", + "native": { + "rms": 0.00040551496122708056, + "max_abs": 0.004686962580308318, + "worst_column_rms": 0.004686962580308318, + "worst_column": 2095, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "original_native": { + "rms": 0.00040551496122708056, + "max_abs": 0.004686962580308318, + "worst_column_rms": 0.004686962580308318, + "worst_column": 2095, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "mlx": { + "rms": 0.00040551496122708056, + "max_abs": 0.004686962580308318, + "worst_column_rms": 0.004686962580308318, + "worst_column": 2095, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "88fe235008e7a04f2aa0a365a599700df32f4e4b511487ae47f84cbd9042147d" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-4bit", + "module": "language_model.model.layers.0.feed_forward.w1", + "input_stage": "layer_00_ffn_norm", + "input_file": "1.7B-4-ffn_gate.bf16", + "one_row_file": "1.7B-4-ffn_gate-one.bf16", + "model_directory": "LFM3-1.7B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 7680 + }, + { + "name": "1.7B-4-ffn_up", + "class": "ffn_up", + "shape": [ + 64, + 2048, + 7680 + ], + "input_sha256": "fa6b92bb7fd858cfc78f98ba720ef3e67984628ffa1c57c7b71834e8a483cfa0", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.0003408959761445204, + "max_abs": 0.006576349260285497, + "worst_column_rms": 0.0023784678000086648, + "worst_column": 3439, + "correctly_rounded": 491520, + "outputs": 491520 + }, + "original_native": { + "rms": 0.0003408959762809756, + "max_abs": 0.006576349260285497, + "worst_column_rms": 0.0023784678000086648, + "worst_column": 3439, + "correctly_rounded": 491470, + "outputs": 491520 + }, + "mlx": { + "rms": 0.000431843641960972, + "max_abs": 0.006576349260285497, + "worst_column_rms": 0.002412187456114085, + "worst_column": 3439, + "correctly_rounded": 302310, + "outputs": 491520 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "bfa00c6ce34772edc11f01c7de19f0b3c9c3b2cf28288500c70cc1a8fe823787" + }, + { + "mode": "decode", + "native": { + "rms": 0.00032491084460863057, + "max_abs": 0.0019470120314508677, + "worst_column_rms": 0.0019470120314508677, + "worst_column": 5343, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "original_native": { + "rms": 0.00032491084460863057, + "max_abs": 0.0019470120314508677, + "worst_column_rms": 0.0019470120314508677, + "worst_column": 5343, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "mlx": { + "rms": 0.00032491084460863057, + "max_abs": 0.0019470120314508677, + "worst_column_rms": 0.0019470120314508677, + "worst_column": 5343, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "7936b781b393ee17409bd70f5bc5592c64963a8d08141633b37e072a3c13362d" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-4bit", + "module": "language_model.model.layers.0.feed_forward.w3", + "input_stage": "layer_00_ffn_norm", + "input_file": "1.7B-4-ffn_up.bf16", + "one_row_file": "1.7B-4-ffn_up-one.bf16", + "model_directory": "LFM3-1.7B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 7680 + }, + { + "name": "1.7B-4-ffn_down", + "class": "ffn_down", + "shape": [ + 64, + 7680, + 2048 + ], + "input_sha256": "ee81b72be970f7c7bfe2f675448b302505c4a89d8cc259b6bfa394eca88566d9", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.0002710084133706065, + "max_abs": 0.018648785813638824, + "worst_column_rms": 0.005861850711822162, + "worst_column": 1402, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 0.00027100841337226, + "max_abs": 0.018648785813638824, + "worst_column_rms": 0.005861850711822162, + "worst_column": 1402, + "correctly_rounded": 131061, + "outputs": 131072 + }, + "mlx": { + "rms": 0.0002822168770496118, + "max_abs": 0.018648785813638824, + "worst_column_rms": 0.005861850711822162, + "worst_column": 1402, + "correctly_rounded": 88906, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "81e564394395e6f5ae3e1ba6f1c524ea76ec070846e90020ae4db3e9a1d8f122" + }, + { + "mode": "decode", + "native": { + "rms": 0.00010020487711375097, + "max_abs": 0.0017595121389604174, + "worst_column_rms": 0.0017595121389604174, + "worst_column": 1688, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 0.00010020487711375097, + "max_abs": 0.0017595121389604174, + "worst_column_rms": 0.0017595121389604174, + "worst_column": 1688, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "mlx": { + "rms": 0.00010020487711375097, + "max_abs": 0.0017595121389604174, + "worst_column_rms": 0.0017595121389604174, + "worst_column": 1688, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "c33b3ac7d6ecc10fc84d4c45adb28d52fd65037a2f34a2293123484fe9747e76" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-4bit", + "module": "language_model.model.layers.0.feed_forward.w2", + "input_stage": "layer_00_act", + "input_file": "1.7B-4-ffn_down.bf16", + "one_row_file": "1.7B-4-ffn_down-one.bf16", + "model_directory": "LFM3-1.7B-MLX-4bit", + "rows": 64, + "k": 7680, + "n": 2048 + }, + { + "name": "1.7B-4-attention_q", + "class": "attention_q", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "b63a3ce95ac63b4c6658f46c28ae63e60b1e7ec8df4523df47ad94dc6844f048", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.0001277586934788277, + "max_abs": 0.0019314179080538452, + "worst_column_rms": 0.0010540158308016817, + "worst_column": 1968, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 0.00012775869350152962, + "max_abs": 0.0019314179080538452, + "worst_column_rms": 0.0010540158308016817, + "worst_column": 1968, + "correctly_rounded": 131060, + "outputs": 131072 + }, + "mlx": { + "rms": 0.00014059075651297226, + "max_abs": 0.001993341647903435, + "worst_column_rms": 0.0010540158308016817, + "worst_column": 1968, + "correctly_rounded": 91943, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "35d010f3010570b13756e02ffad15c2e3d88a20c8fa17d848fc9644e7e46f41f" + }, + { + "mode": "decode", + "native": { + "rms": 6.896103597185678e-05, + "max_abs": 0.0005935978260822594, + "worst_column_rms": 0.0005935978260822594, + "worst_column": 1968, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 6.896103597185678e-05, + "max_abs": 0.0005935978260822594, + "worst_column_rms": 0.0005935978260822594, + "worst_column": 1968, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "mlx": { + "rms": 6.896103597185678e-05, + "max_abs": 0.0005935978260822594, + "worst_column_rms": 0.0005935978260822594, + "worst_column": 1968, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "a2a0743f91f9e9b823701b82fd82860e202ad3a82b6c94cd8b8a200f5a48f86c" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-4bit", + "module": "language_model.model.layers.2.self_attn.q_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "1.7B-4-attention_q.bf16", + "one_row_file": "1.7B-4-attention_q-one.bf16", + "model_directory": "LFM3-1.7B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 2048 + }, + { + "name": "1.7B-4-attention_k", + "class": "attention_k", + "shape": [ + 64, + 2048, + 512 + ], + "input_sha256": "b63a3ce95ac63b4c6658f46c28ae63e60b1e7ec8df4523df47ad94dc6844f048", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.00012784894910553032, + "max_abs": 0.0016299454437103122, + "worst_column_rms": 0.0006007459212056101, + "worst_column": 223, + "correctly_rounded": 32768, + "outputs": 32768 + }, + "original_native": { + "rms": 0.00012784894910553032, + "max_abs": 0.0016299454437103122, + "worst_column_rms": 0.0006007459212056101, + "worst_column": 223, + "correctly_rounded": 32768, + "outputs": 32768 + }, + "mlx": { + "rms": 0.00014046471841790305, + "max_abs": 0.0016299454437103122, + "worst_column_rms": 0.0006139709098557164, + "worst_column": 223, + "correctly_rounded": 23806, + "outputs": 32768 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "b6119e6fdfa2e6b2e57e63a5efbdde3777ea84cebadd42f70b8eb2d24a0b38c6" + }, + { + "mode": "decode", + "native": { + "rms": 8.182879240875662e-05, + "max_abs": 0.0006744395577698015, + "worst_column_rms": 0.0006744395577698015, + "worst_column": 103, + "correctly_rounded": 512, + "outputs": 512 + }, + "original_native": { + "rms": 8.182879240875662e-05, + "max_abs": 0.0006744395577698015, + "worst_column_rms": 0.0006744395577698015, + "worst_column": 103, + "correctly_rounded": 512, + "outputs": 512 + }, + "mlx": { + "rms": 8.182879240875662e-05, + "max_abs": 0.0006744395577698015, + "worst_column_rms": 0.0006744395577698015, + "worst_column": 103, + "correctly_rounded": 512, + "outputs": 512 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "49cd1a700b8e55f3ae9081f9dfec38426a2c42cfd091122200705bc882dde26e" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-4bit", + "module": "language_model.model.layers.2.self_attn.k_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "1.7B-4-attention_k.bf16", + "one_row_file": "1.7B-4-attention_k-one.bf16", + "model_directory": "LFM3-1.7B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 512 + }, + { + "name": "1.7B-4-attention_v", + "class": "attention_v", + "shape": [ + 64, + 2048, + 512 + ], + "input_sha256": "b63a3ce95ac63b4c6658f46c28ae63e60b1e7ec8df4523df47ad94dc6844f048", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 9.758553227201483e-05, + "max_abs": 0.0018437682010699064, + "worst_column_rms": 0.0005222599323188061, + "worst_column": 87, + "correctly_rounded": 32768, + "outputs": 32768 + }, + "original_native": { + "rms": 9.758553234604846e-05, + "max_abs": 0.0018437682010699064, + "worst_column_rms": 0.0005222599323188061, + "worst_column": 87, + "correctly_rounded": 32755, + "outputs": 32768 + }, + "mlx": { + "rms": 0.00011400362912372623, + "max_abs": 0.0018437682010699064, + "worst_column_rms": 0.0005241125791090504, + "worst_column": 87, + "correctly_rounded": 21636, + "outputs": 32768 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "c8e358a2139f014b68ef723b2ad0b600422c1600c93dd497e42818487a4cf8b1" + }, + { + "mode": "decode", + "native": { + "rms": 6.693646406130574e-05, + "max_abs": 0.0006725182829541154, + "worst_column_rms": 0.0006725182829541154, + "worst_column": 92, + "correctly_rounded": 512, + "outputs": 512 + }, + "original_native": { + "rms": 6.693646406148605e-05, + "max_abs": 0.0006725182829541154, + "worst_column_rms": 0.0006725182829541154, + "worst_column": 92, + "correctly_rounded": 511, + "outputs": 512 + }, + "mlx": { + "rms": 6.693646406148605e-05, + "max_abs": 0.0006725182829541154, + "worst_column_rms": 0.0006725182829541154, + "worst_column": 92, + "correctly_rounded": 511, + "outputs": 512 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "f807cedaf1556dc08feeb85ab4e26c8e9adb2597a9a27f7cce06259b40e95643" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-4bit", + "module": "language_model.model.layers.2.self_attn.v_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "1.7B-4-attention_v.bf16", + "one_row_file": "1.7B-4-attention_v-one.bf16", + "model_directory": "LFM3-1.7B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 512 + }, + { + "name": "1.7B-4-attention_gate", + "class": "attention_gate", + "shape": [ + 64, + 2048, + 16 + ], + "input_sha256": "b63a3ce95ac63b4c6658f46c28ae63e60b1e7ec8df4523df47ad94dc6844f048", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.0009976975410925608, + "max_abs": 0.003880350152030587, + "worst_column_rms": 0.0017390733495844682, + "worst_column": 13, + "correctly_rounded": 1024, + "outputs": 1024 + }, + "original_native": { + "rms": 0.0009976975410925608, + "max_abs": 0.003880350152030587, + "worst_column_rms": 0.0017390733495844682, + "worst_column": 13, + "correctly_rounded": 1024, + "outputs": 1024 + }, + "mlx": { + "rms": 0.00104562658331415, + "max_abs": 0.003880350152030587, + "worst_column_rms": 0.0018090669031358806, + "worst_column": 13, + "correctly_rounded": 841, + "outputs": 1024 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "acaf96d0437193e28abe706219a35cd47abb6783f846a6c3edcf376c73ebbbd5" + }, + { + "mode": "decode", + "native": { + "rms": 0.0002541051762510186, + "max_abs": 0.0005612098902929574, + "worst_column_rms": 0.0005612098902929574, + "worst_column": 14, + "correctly_rounded": 16, + "outputs": 16 + }, + "original_native": { + "rms": 0.0002541051762510186, + "max_abs": 0.0005612098902929574, + "worst_column_rms": 0.0005612098902929574, + "worst_column": 14, + "correctly_rounded": 16, + "outputs": 16 + }, + "mlx": { + "rms": 0.0002541051762510186, + "max_abs": 0.0005612098902929574, + "worst_column_rms": 0.0005612098902929574, + "worst_column": 14, + "correctly_rounded": 16, + "outputs": 16 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "04c0a5e4019680d000290cddf3e2a236028ab4f341059eab523bab6f452402d3" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-4bit", + "module": "language_model.model.layers.2.self_attn.gate_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "1.7B-4-attention_gate.bf16", + "one_row_file": "1.7B-4-attention_gate-one.bf16", + "model_directory": "LFM3-1.7B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 16 + }, + { + "name": "1.7B-4-attention_out", + "class": "attention_out", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "dd9dbbb7243c8cab2e140aaf01b3e578a9e05940f1295b3f18ea21dd1f3f28e8", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 3.3805597927010204e-05, + "max_abs": 0.0009752579353516921, + "worst_column_rms": 0.00039015448467111767, + "worst_column": 1777, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 3.380559793139201e-05, + "max_abs": 0.0009752579353516921, + "worst_column_rms": 0.00039015448467111767, + "worst_column": 1777, + "correctly_rounded": 131064, + "outputs": 131072 + }, + "mlx": { + "rms": 4.001960300054414e-05, + "max_abs": 0.0010739647696027532, + "worst_column_rms": 0.0004094830318981674, + "worst_column": 1777, + "correctly_rounded": 81412, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "fcc3661b25c7a9b58e7cf37790af16a9f9775061dc504e8880574cf5b3064509" + }, + { + "mode": "decode", + "native": { + "rms": 3.0352825200527908e-05, + "max_abs": 0.000475818878840073, + "worst_column_rms": 0.000475818878840073, + "worst_column": 236, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 3.0352825205494347e-05, + "max_abs": 0.000475818878840073, + "worst_column_rms": 0.000475818878840073, + "worst_column": 236, + "correctly_rounded": 2047, + "outputs": 2048 + }, + "mlx": { + "rms": 3.0352825205494347e-05, + "max_abs": 0.000475818878840073, + "worst_column_rms": 0.000475818878840073, + "worst_column": 236, + "correctly_rounded": 2047, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "9bc91b6132035b4c9c3e26eb2b0e1ff0471e8b1675f0de175c4555e4e3e4a6d3" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-4bit", + "module": "language_model.model.layers.2.self_attn.o_proj", + "input_stage": "layer_02_attn_gated", + "input_file": "1.7B-4-attention_out.bf16", + "one_row_file": "1.7B-4-attention_out-one.bf16", + "model_directory": "LFM3-1.7B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 2048 + }, + { + "name": "1.7B-4-vocabulary_head", + "class": "vocabulary_head", + "shape": [ + 64, + 2048, + 128000 + ], + "input_sha256": "094c217196e7df4e96d3462a589ba59b45f88c7921f347f1795595709be512a3", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.004679167671913785, + "max_abs": 0.06246214359998703, + "worst_column_rms": 0.026925865351463595, + "worst_column": 309, + "correctly_rounded": 8192000, + "outputs": 8192000 + }, + "original_native": { + "rms": 0.004679167676166402, + "max_abs": 0.06246214359998703, + "worst_column_rms": 0.026925865351463595, + "worst_column": 309, + "correctly_rounded": 8191400, + "outputs": 8192000 + }, + "mlx": { + "rms": 0.005303761359707796, + "max_abs": 0.0663363141939044, + "worst_column_rms": 0.027180539243717303, + "worst_column": 309, + "correctly_rounded": 5743067, + "outputs": 8192000 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "6786268290e834bbc1d6f7367f17f3d8240fede1a1c66608edf843bdd3ddd3b9" + }, + { + "mode": "decode", + "native": { + "rms": 0.003644296309715534, + "max_abs": 0.031142726074904203, + "worst_column_rms": 0.031142726074904203, + "worst_column": 16992, + "correctly_rounded": 128000, + "outputs": 128000 + }, + "original_native": { + "rms": 0.0036442963168938224, + "max_abs": 0.031142726074904203, + "worst_column_rms": 0.031142726074904203, + "worst_column": 16992, + "correctly_rounded": 127989, + "outputs": 128000 + }, + "mlx": { + "rms": 0.0036442963168938224, + "max_abs": 0.031142726074904203, + "worst_column_rms": 0.031142726074904203, + "worst_column": 16992, + "correctly_rounded": 127989, + "outputs": 128000 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "ab9a7a2a9c4ce1da97742a4d0d5bfd497ee5f97be6e7f40ddbb35074779490f1" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-4bit", + "module": "language_model.model.embed_tokens", + "input_stage": "final_norm", + "input_file": "1.7B-4-vocabulary_head.bf16", + "one_row_file": "1.7B-4-vocabulary_head-one.bf16", + "model_directory": "LFM3-1.7B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 128000 + }, + { + "name": "1.7B-6-conv_in", + "class": "conv_in", + "shape": [ + 64, + 2048, + 6144 + ], + "input_sha256": "e54b7ab3e9d5a7b02307b8cca254d410550e9e470a5134773258eb5eb3336e94", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.00044103478884401244, + "max_abs": 0.007179600885137916, + "worst_column_rms": 0.0026935385287125265, + "worst_column": 122, + "correctly_rounded": 393216, + "outputs": 393216 + }, + "original_native": { + "rms": 0.0004410347894187434, + "max_abs": 0.007179600885137916, + "worst_column_rms": 0.0026935385287125265, + "worst_column": 122, + "correctly_rounded": 393184, + "outputs": 393216 + }, + "mlx": { + "rms": 0.000526270327263699, + "max_abs": 0.007179600885137916, + "worst_column_rms": 0.0026935385287125265, + "worst_column": 122, + "correctly_rounded": 232755, + "outputs": 393216 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "e690ade98b46e3c1ed023f33a6c729070b280b6b367b6356d046bad571f9e7d5" + }, + { + "mode": "decode", + "native": { + "rms": 0.0004857510246401321, + "max_abs": 0.003867839368467685, + "worst_column_rms": 0.003867839368467685, + "worst_column": 2582, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "original_native": { + "rms": 0.0004857510246401321, + "max_abs": 0.003867839368467685, + "worst_column_rms": 0.003867839368467685, + "worst_column": 2582, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "mlx": { + "rms": 0.0004857510246401321, + "max_abs": 0.003867839368467685, + "worst_column_rms": 0.003867839368467685, + "worst_column": 2582, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "8812effabc5786e48e33c14f58aa0aed55ba1a2041ff8c68aa7c0e57c1d5db14" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-6bit", + "module": "language_model.model.layers.0.conv.in_proj", + "input_stage": "layer_00_operator_norm", + "input_file": "1.7B-6-conv_in.bf16", + "one_row_file": "1.7B-6-conv_in-one.bf16", + "model_directory": "LFM3-1.7B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 6144 + }, + { + "name": "1.7B-6-conv_out", + "class": "conv_out", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "394b33af5ddf31365ddd9bc9b02dcfa775665faa3332eb3a6bdcf07b93778905", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 1.5808096750977904e-05, + "max_abs": 0.0009582132712466773, + "worst_column_rms": 0.00029336551703701137, + "worst_column": 1688, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 1.580809675370336e-05, + "max_abs": 0.0009582132712466773, + "worst_column_rms": 0.00029336551703701137, + "worst_column": 1688, + "correctly_rounded": 131052, + "outputs": 131072 + }, + "mlx": { + "rms": 1.717138809360215e-05, + "max_abs": 0.0009582132712466773, + "worst_column_rms": 0.0002934092223049504, + "worst_column": 1688, + "correctly_rounded": 90404, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "7b42d71bdf4c84a41f239b9928d62a42d591e1a19a32595add3b17bb0402ec08" + }, + { + "mode": "decode", + "native": { + "rms": 2.3766273888193977e-06, + "max_abs": 1.707213518833961e-05, + "worst_column_rms": 1.707213518833961e-05, + "worst_column": 1688, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 2.3766273888193977e-06, + "max_abs": 1.707213518833961e-05, + "worst_column_rms": 1.707213518833961e-05, + "worst_column": 1688, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "mlx": { + "rms": 2.3766273888193977e-06, + "max_abs": 1.707213518833961e-05, + "worst_column_rms": 1.707213518833961e-05, + "worst_column": 1688, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "6b6acdf7bfcca8b64f078b9bc8ce48a03490595f2870672482e1509380bcd107" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-6bit", + "module": "language_model.model.layers.0.conv.out_proj", + "input_stage": "layer_00_conv_gated", + "input_file": "1.7B-6-conv_out.bf16", + "one_row_file": "1.7B-6-conv_out-one.bf16", + "model_directory": "LFM3-1.7B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 2048 + }, + { + "name": "1.7B-6-ffn_gate", + "class": "ffn_gate", + "shape": [ + 64, + 2048, + 7680 + ], + "input_sha256": "82800362eddb8dac0e354b497bfa2ab57424bfdbb25017041730fed6aea95d77", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.0005471777301209168, + "max_abs": 0.007793171884259209, + "worst_column_rms": 0.00462051108529706, + "worst_column": 6325, + "correctly_rounded": 491520, + "outputs": 491520 + }, + "original_native": { + "rms": 0.0005471777301897714, + "max_abs": 0.007793171884259209, + "worst_column_rms": 0.00462051108529706, + "worst_column": 6325, + "correctly_rounded": 491481, + "outputs": 491520 + }, + "mlx": { + "rms": 0.0006373414300532488, + "max_abs": 0.008380344370380044, + "worst_column_rms": 0.004719592405840302, + "worst_column": 6325, + "correctly_rounded": 334888, + "outputs": 491520 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "6ff356e3c153ebc2efebc895be224be29f28512a97f76eaee2e29b20c56aab32" + }, + { + "mode": "decode", + "native": { + "rms": 0.0004216263782380626, + "max_abs": 0.007743538822978735, + "worst_column_rms": 0.007743538822978735, + "worst_column": 3381, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "original_native": { + "rms": 0.00042162637824663413, + "max_abs": 0.007743538822978735, + "worst_column_rms": 0.007743538822978735, + "worst_column": 3381, + "correctly_rounded": 7679, + "outputs": 7680 + }, + "mlx": { + "rms": 0.00042162637824663413, + "max_abs": 0.007743538822978735, + "worst_column_rms": 0.007743538822978735, + "worst_column": 3381, + "correctly_rounded": 7679, + "outputs": 7680 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "dc997a64e86ef2c76b962890db232dddd6bdd55819092a846abd0b0f5b82e6f3" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-6bit", + "module": "language_model.model.layers.0.feed_forward.w1", + "input_stage": "layer_00_ffn_norm", + "input_file": "1.7B-6-ffn_gate.bf16", + "one_row_file": "1.7B-6-ffn_gate-one.bf16", + "model_directory": "LFM3-1.7B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 7680 + }, + { + "name": "1.7B-6-ffn_up", + "class": "ffn_up", + "shape": [ + 64, + 2048, + 7680 + ], + "input_sha256": "82800362eddb8dac0e354b497bfa2ab57424bfdbb25017041730fed6aea95d77", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.00034296867819688345, + "max_abs": 0.007138871122151613, + "worst_column_rms": 0.0025097931218887545, + "worst_column": 445, + "correctly_rounded": 491520, + "outputs": 491520 + }, + "original_native": { + "rms": 0.00034296867892932186, + "max_abs": 0.007138871122151613, + "worst_column_rms": 0.0025097931218887545, + "worst_column": 445, + "correctly_rounded": 491475, + "outputs": 491520 + }, + "mlx": { + "rms": 0.0004458432497103546, + "max_abs": 0.007138871122151613, + "worst_column_rms": 0.002538975713541404, + "worst_column": 445, + "correctly_rounded": 293876, + "outputs": 491520 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "ad79c7c7bfd66a7545a9d76490566a10a3fd3dd0ee2fd74f97ca0e144a52f2f4" + }, + { + "mode": "decode", + "native": { + "rms": 0.0003180742396938846, + "max_abs": 0.002300340449437499, + "worst_column_rms": 0.002300340449437499, + "worst_column": 1409, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "original_native": { + "rms": 0.00031807424020581764, + "max_abs": 0.002300340449437499, + "worst_column_rms": 0.002300340449437499, + "worst_column": 1409, + "correctly_rounded": 7679, + "outputs": 7680 + }, + "mlx": { + "rms": 0.00031807424020581764, + "max_abs": 0.002300340449437499, + "worst_column_rms": 0.002300340449437499, + "worst_column": 1409, + "correctly_rounded": 7679, + "outputs": 7680 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "546cd059b084c49f08ca8244d16b9ee8a5c2c4c797492adf45d359cb77ac141a" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-6bit", + "module": "language_model.model.layers.0.feed_forward.w3", + "input_stage": "layer_00_ffn_norm", + "input_file": "1.7B-6-ffn_up.bf16", + "one_row_file": "1.7B-6-ffn_up-one.bf16", + "model_directory": "LFM3-1.7B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 7680 + }, + { + "name": "1.7B-6-ffn_down", + "class": "ffn_down", + "shape": [ + 64, + 7680, + 2048 + ], + "input_sha256": "c21481e298f440f38359438a5492d7c439bbf6c68637ffc1bfeafb928bd631e2", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.00028479512307101796, + "max_abs": 0.023310194078987934, + "worst_column_rms": 0.0065849927185953755, + "worst_column": 108, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 0.0002847951230889489, + "max_abs": 0.023310194078987934, + "worst_column_rms": 0.0065849927185953755, + "worst_column": 108, + "correctly_rounded": 131044, + "outputs": 131072 + }, + "mlx": { + "rms": 0.000296974424946525, + "max_abs": 0.023310194078987934, + "worst_column_rms": 0.0065849927185953755, + "worst_column": 108, + "correctly_rounded": 86025, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "3a6b7f54b9e6e33077d0bc94526b59f8bd460b4ef08470658b17df388abf6789" + }, + { + "mode": "decode", + "native": { + "rms": 0.00011031410953777374, + "max_abs": 0.0018526755170569231, + "worst_column_rms": 0.0018526755170569231, + "worst_column": 1688, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 0.00011031410953777374, + "max_abs": 0.0018526755170569231, + "worst_column_rms": 0.0018526755170569231, + "worst_column": 1688, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "mlx": { + "rms": 0.00011031410953777374, + "max_abs": 0.0018526755170569231, + "worst_column_rms": 0.0018526755170569231, + "worst_column": 1688, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "d58d233e1dd4097bce3dda862fee878e15ffa51016c11d235f7544a7cc6e3db3" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-6bit", + "module": "language_model.model.layers.0.feed_forward.w2", + "input_stage": "layer_00_act", + "input_file": "1.7B-6-ffn_down.bf16", + "one_row_file": "1.7B-6-ffn_down-one.bf16", + "model_directory": "LFM3-1.7B-MLX-6bit", + "rows": 64, + "k": 7680, + "n": 2048 + }, + { + "name": "1.7B-6-attention_q", + "class": "attention_q", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "cecf2508e4de61c6e75839e1c87764bcdae093d05d1f29d03831db1e476f01b3", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.00012942290487210885, + "max_abs": 0.0019434237547102384, + "worst_column_rms": 0.001227132353580726, + "worst_column": 1968, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 0.00012942290506512676, + "max_abs": 0.0019434237547102384, + "worst_column_rms": 0.001227132353580726, + "worst_column": 1968, + "correctly_rounded": 131057, + "outputs": 131072 + }, + "mlx": { + "rms": 0.00014425094378050992, + "max_abs": 0.0019898611353710294, + "worst_column_rms": 0.0012315115498758471, + "worst_column": 1968, + "correctly_rounded": 89097, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "20779bc07a475a9ca2e2e43f9dfcc20e5e335854cb233f1633a45d8fa425140e" + }, + { + "mode": "decode", + "native": { + "rms": 6.976904949205191e-05, + "max_abs": 0.00047946947961463593, + "worst_column_rms": 0.00047946947961463593, + "worst_column": 1592, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 6.976904949205191e-05, + "max_abs": 0.00047946947961463593, + "worst_column_rms": 0.00047946947961463593, + "worst_column": 1592, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "mlx": { + "rms": 6.976904949205191e-05, + "max_abs": 0.00047946947961463593, + "worst_column_rms": 0.00047946947961463593, + "worst_column": 1592, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "0bacb201fc20be38fbe4bc2589867d82c517ee19f3c261766f80d0f1f80c1cbf" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-6bit", + "module": "language_model.model.layers.2.self_attn.q_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "1.7B-6-attention_q.bf16", + "one_row_file": "1.7B-6-attention_q-one.bf16", + "model_directory": "LFM3-1.7B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 2048 + }, + { + "name": "1.7B-6-attention_k", + "class": "attention_k", + "shape": [ + 64, + 2048, + 512 + ], + "input_sha256": "cecf2508e4de61c6e75839e1c87764bcdae093d05d1f29d03831db1e476f01b3", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.00013021484447229686, + "max_abs": 0.001910699542349903, + "worst_column_rms": 0.0007017638707859753, + "worst_column": 158, + "correctly_rounded": 32768, + "outputs": 32768 + }, + "original_native": { + "rms": 0.00013021484457899653, + "max_abs": 0.001910699542349903, + "worst_column_rms": 0.0007017638707859753, + "worst_column": 158, + "correctly_rounded": 32764, + "outputs": 32768 + }, + "mlx": { + "rms": 0.00014430660011154577, + "max_abs": 0.001910699542349903, + "worst_column_rms": 0.0007027963194872706, + "worst_column": 158, + "correctly_rounded": 23238, + "outputs": 32768 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "92fef97597874698bad91388b36fcb1579fd9176643637397e43bca03a61d4e0" + }, + { + "mode": "decode", + "native": { + "rms": 9.396400087087372e-05, + "max_abs": 0.0009199798496410949, + "worst_column_rms": 0.0009199798496410949, + "worst_column": 487, + "correctly_rounded": 512, + "outputs": 512 + }, + "original_native": { + "rms": 9.396400442934229e-05, + "max_abs": 0.0009199798496410949, + "worst_column_rms": 0.0009199798496410949, + "worst_column": 487, + "correctly_rounded": 511, + "outputs": 512 + }, + "mlx": { + "rms": 9.396400087087372e-05, + "max_abs": 0.0009199798496410949, + "worst_column_rms": 0.0009199798496410949, + "worst_column": 487, + "correctly_rounded": 512, + "outputs": 512 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "b59457203b990344119a76037c9ad8e4dbeaaa005239ef3d6c12d1dad72d229f" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-6bit", + "module": "language_model.model.layers.2.self_attn.k_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "1.7B-6-attention_k.bf16", + "one_row_file": "1.7B-6-attention_k-one.bf16", + "model_directory": "LFM3-1.7B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 512 + }, + { + "name": "1.7B-6-attention_v", + "class": "attention_v", + "shape": [ + 64, + 2048, + 512 + ], + "input_sha256": "cecf2508e4de61c6e75839e1c87764bcdae093d05d1f29d03831db1e476f01b3", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 9.882400418830994e-05, + "max_abs": 0.001639702375541674, + "worst_column_rms": 0.00040804254931952743, + "worst_column": 233, + "correctly_rounded": 32768, + "outputs": 32768 + }, + "original_native": { + "rms": 9.882400418831352e-05, + "max_abs": 0.001639702375541674, + "worst_column_rms": 0.00040804254931952743, + "worst_column": 233, + "correctly_rounded": 32766, + "outputs": 32768 + }, + "mlx": { + "rms": 0.00011883640791045365, + "max_abs": 0.001639702375541674, + "worst_column_rms": 0.0004178019741907418, + "worst_column": 233, + "correctly_rounded": 20807, + "outputs": 32768 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "087f15e21a21ff48959028377610106d96c0bfd2f78e37ce6f039eaea7f95fb2" + }, + { + "mode": "decode", + "native": { + "rms": 6.147290792581313e-05, + "max_abs": 0.0003950981326852343, + "worst_column_rms": 0.0003950981326852343, + "worst_column": 92, + "correctly_rounded": 512, + "outputs": 512 + }, + "original_native": { + "rms": 6.147290792581313e-05, + "max_abs": 0.0003950981326852343, + "worst_column_rms": 0.0003950981326852343, + "worst_column": 92, + "correctly_rounded": 512, + "outputs": 512 + }, + "mlx": { + "rms": 6.147290792581313e-05, + "max_abs": 0.0003950981326852343, + "worst_column_rms": 0.0003950981326852343, + "worst_column": 92, + "correctly_rounded": 512, + "outputs": 512 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "3b40c61d73782c5b2193028ea246f7964508017bf74fbf63e69750c99edce57a" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-6bit", + "module": "language_model.model.layers.2.self_attn.v_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "1.7B-6-attention_v.bf16", + "one_row_file": "1.7B-6-attention_v-one.bf16", + "model_directory": "LFM3-1.7B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 512 + }, + { + "name": "1.7B-6-attention_gate", + "class": "attention_gate", + "shape": [ + 64, + 2048, + 16 + ], + "input_sha256": "cecf2508e4de61c6e75839e1c87764bcdae093d05d1f29d03831db1e476f01b3", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.001063476022120072, + "max_abs": 0.003867838007863611, + "worst_column_rms": 0.0018406449464999919, + "worst_column": 13, + "correctly_rounded": 1024, + "outputs": 1024 + }, + "original_native": { + "rms": 0.001063476022120072, + "max_abs": 0.003867838007863611, + "worst_column_rms": 0.0018406449464999919, + "worst_column": 13, + "correctly_rounded": 1024, + "outputs": 1024 + }, + "mlx": { + "rms": 0.0011232387998164122, + "max_abs": 0.005117520602652803, + "worst_column_rms": 0.0020809571783034424, + "worst_column": 13, + "correctly_rounded": 828, + "outputs": 1024 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "a71bd4049b62100ef10b401077aa7f1ee7b92cf39165bf06f217847b417f08c4" + }, + { + "mode": "decode", + "native": { + "rms": 0.0005059635356628884, + "max_abs": 0.001740223124215845, + "worst_column_rms": 0.001740223124215845, + "worst_column": 13, + "correctly_rounded": 16, + "outputs": 16 + }, + "original_native": { + "rms": 0.0005059635356628884, + "max_abs": 0.001740223124215845, + "worst_column_rms": 0.001740223124215845, + "worst_column": 13, + "correctly_rounded": 16, + "outputs": 16 + }, + "mlx": { + "rms": 0.0005059635356628884, + "max_abs": 0.001740223124215845, + "worst_column_rms": 0.001740223124215845, + "worst_column": 13, + "correctly_rounded": 16, + "outputs": 16 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "bd5cc9257542ff115afb330579f0fc9ca115c7b01bd446b1059f8e4ecbf2bba0" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-6bit", + "module": "language_model.model.layers.2.self_attn.gate_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "1.7B-6-attention_gate.bf16", + "one_row_file": "1.7B-6-attention_gate-one.bf16", + "model_directory": "LFM3-1.7B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 16 + }, + { + "name": "1.7B-6-attention_out", + "class": "attention_out", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "ce22331702ccb8e91536b79c7fc115a69c8fb075aa7ca0d9f89649ab26ebcbf0", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 3.4456098682593906e-05, + "max_abs": 0.0009694274267531, + "worst_column_rms": 0.00044306484434214515, + "worst_column": 1918, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 3.445609868780154e-05, + "max_abs": 0.0009694274267531, + "worst_column_rms": 0.00044306484434214515, + "worst_column": 1918, + "correctly_rounded": 131050, + "outputs": 131072 + }, + "mlx": { + "rms": 4.18432521128754e-05, + "max_abs": 0.001253318478120491, + "worst_column_rms": 0.0004836698924784144, + "worst_column": 1918, + "correctly_rounded": 78660, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "924ec41f566032e6928c145693aef559153b4cc4e0e3582002ca4cb3f1bb5574" + }, + { + "mode": "decode", + "native": { + "rms": 2.8833834168727128e-05, + "max_abs": 0.00043793422082671896, + "worst_column_rms": 0.00043793422082671896, + "worst_column": 1996, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 2.8833834168858977e-05, + "max_abs": 0.00043793422082671896, + "worst_column_rms": 0.00043793422082671896, + "worst_column": 1996, + "correctly_rounded": 2047, + "outputs": 2048 + }, + "mlx": { + "rms": 2.8833834168727128e-05, + "max_abs": 0.00043793422082671896, + "worst_column_rms": 0.00043793422082671896, + "worst_column": 1996, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "53b6cdfc5c6ae2b1989b3250cb5fe665338d00065a9f340ee99b7878d3c796f9" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-6bit", + "module": "language_model.model.layers.2.self_attn.o_proj", + "input_stage": "layer_02_attn_gated", + "input_file": "1.7B-6-attention_out.bf16", + "one_row_file": "1.7B-6-attention_out-one.bf16", + "model_directory": "LFM3-1.7B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 2048 + }, + { + "name": "1.7B-6-vocabulary_head", + "class": "vocabulary_head", + "shape": [ + 64, + 2048, + 128000 + ], + "input_sha256": "24b6bed4a040f4a9780c78e325e1fec9b07a9c257c0c71ba9782424f600401c5", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.004710796712846788, + "max_abs": 0.06246721709612757, + "worst_column_rms": 0.027752774356482587, + "worst_column": 278, + "correctly_rounded": 8192000, + "outputs": 8192000 + }, + "original_native": { + "rms": 0.004710796719485594, + "max_abs": 0.06246721709612757, + "worst_column_rms": 0.027752774356482587, + "worst_column": 278, + "correctly_rounded": 8191355, + "outputs": 8192000 + }, + "mlx": { + "rms": 0.00542013568523091, + "max_abs": 0.06862405920401216, + "worst_column_rms": 0.02840478222398789, + "worst_column": 278, + "correctly_rounded": 5629148, + "outputs": 8192000 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "4aa176ed5fda5e148f8ae28a9aa7ab2476b91e447a1eb84d3c543a20f9fc043b" + }, + { + "mode": "decode", + "native": { + "rms": 0.0038446951022122946, + "max_abs": 0.029980748426169157, + "worst_column_rms": 0.029980748426169157, + "worst_column": 2069, + "correctly_rounded": 128000, + "outputs": 128000 + }, + "original_native": { + "rms": 0.0038446951050451826, + "max_abs": 0.029980748426169157, + "worst_column_rms": 0.029980748426169157, + "worst_column": 2069, + "correctly_rounded": 127987, + "outputs": 128000 + }, + "mlx": { + "rms": 0.003844695102317483, + "max_abs": 0.029980748426169157, + "worst_column_rms": 0.029980748426169157, + "worst_column": 2069, + "correctly_rounded": 127989, + "outputs": 128000 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "a7d12df1673fad09167e8ff16292935fc0eec2044e57119bdd14c9cf7ad914c1" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-6bit", + "module": "language_model.model.embed_tokens", + "input_stage": "final_norm", + "input_file": "1.7B-6-vocabulary_head.bf16", + "one_row_file": "1.7B-6-vocabulary_head-one.bf16", + "model_directory": "LFM3-1.7B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 128000 + }, + { + "name": "1.7B-8-conv_in", + "class": "conv_in", + "shape": [ + 64, + 2048, + 6144 + ], + "input_sha256": "c7a8efaf9dbf523fa99251dcda1fe460d07d390dc2511a5210648e7635b6a07e", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.000443545236073818, + "max_abs": 0.007801041952916421, + "worst_column_rms": 0.0031588155218771623, + "worst_column": 3716, + "correctly_rounded": 393216, + "outputs": 393216 + }, + "original_native": { + "rms": 0.00044354523633282234, + "max_abs": 0.007801041952916421, + "worst_column_rms": 0.0031588155218771623, + "worst_column": 3716, + "correctly_rounded": 393151, + "outputs": 393216 + }, + "mlx": { + "rms": 0.0005331741557186632, + "max_abs": 0.007823958047083579, + "worst_column_rms": 0.003184308644523379, + "worst_column": 3716, + "correctly_rounded": 230906, + "outputs": 393216 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "d6b35c4877de2f0a8d67cd1f068b9826d6ee1529c653dbf90fa542ef8f2636ae" + }, + { + "mode": "decode", + "native": { + "rms": 0.0005062529740461505, + "max_abs": 0.003808464120083954, + "worst_column_rms": 0.003808464120083954, + "worst_column": 1788, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "original_native": { + "rms": 0.0005062529740462367, + "max_abs": 0.003808464120083954, + "worst_column_rms": 0.003808464120083954, + "worst_column": 1788, + "correctly_rounded": 6143, + "outputs": 6144 + }, + "mlx": { + "rms": 0.0005062529740461505, + "max_abs": 0.003808464120083954, + "worst_column_rms": 0.003808464120083954, + "worst_column": 1788, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "7e9ceb9220c05a6b5069aee118ad7bff5449da2d9e91c9ff8c5eea17cf7263e8" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-8bit", + "module": "language_model.model.layers.0.conv.in_proj", + "input_stage": "layer_00_operator_norm", + "input_file": "1.7B-8-conv_in.bf16", + "one_row_file": "1.7B-8-conv_in-one.bf16", + "model_directory": "LFM3-1.7B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 6144 + }, + { + "name": "1.7B-8-conv_out", + "class": "conv_out", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "2e9d44acb3e83de632ea8358931e316e735fe92e3e40af08d4d81e35b4e4b0df", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 1.5748016386287074e-05, + "max_abs": 0.0009415286645721999, + "worst_column_rms": 0.0003122769363857758, + "worst_column": 42, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 1.574801641047211e-05, + "max_abs": 0.0009415286645721999, + "worst_column_rms": 0.0003122769363857758, + "worst_column": 42, + "correctly_rounded": 131047, + "outputs": 131072 + }, + "mlx": { + "rms": 1.7151588187688553e-05, + "max_abs": 0.0010115963354278001, + "worst_column_rms": 0.0003190508991324597, + "worst_column": 42, + "correctly_rounded": 89207, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "f1469364d1355c14c380655d42e4aef357c54ee58a03f767e65073c69bcc77db" + }, + { + "mode": "decode", + "native": { + "rms": 2.6253251056775836e-06, + "max_abs": 4.3876711825902026e-05, + "worst_column_rms": 4.3876711825902026e-05, + "worst_column": 1700, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 2.6253251056775836e-06, + "max_abs": 4.3876711825902026e-05, + "worst_column_rms": 4.3876711825902026e-05, + "worst_column": 1700, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "mlx": { + "rms": 2.6253251056775836e-06, + "max_abs": 4.3876711825902026e-05, + "worst_column_rms": 4.3876711825902026e-05, + "worst_column": 1700, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "347e2cee0ad33b3cf957d1487e3fd2200eed2b6f4de206da8f2b0758ab471c86" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-8bit", + "module": "language_model.model.layers.0.conv.out_proj", + "input_stage": "layer_00_conv_gated", + "input_file": "1.7B-8-conv_out.bf16", + "one_row_file": "1.7B-8-conv_out-one.bf16", + "model_directory": "LFM3-1.7B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 2048 + }, + { + "name": "1.7B-8-ffn_gate", + "class": "ffn_gate", + "shape": [ + 64, + 2048, + 7680 + ], + "input_sha256": "0ae00693590fcb04ffdf3d2b3b3e4f4781fc2040ba805edd75eb4fe43d02462e", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.000546315738232988, + "max_abs": 0.007732816913630813, + "worst_column_rms": 0.004794713896403656, + "worst_column": 6325, + "correctly_rounded": 491520, + "outputs": 491520 + }, + "original_native": { + "rms": 0.000546315738615915, + "max_abs": 0.007732816913630813, + "worst_column_rms": 0.004794713896403656, + "worst_column": 6325, + "correctly_rounded": 491483, + "outputs": 491520 + }, + "mlx": { + "rms": 0.0006403613040038655, + "max_abs": 0.008467245905194432, + "worst_column_rms": 0.004841424076967674, + "worst_column": 6325, + "correctly_rounded": 332478, + "outputs": 491520 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "3359e636ffbda8f5c9a3267435991cca16cc88e19b4919f566a4c3d3a96a90cd" + }, + { + "mode": "decode", + "native": { + "rms": 0.00041784331596165115, + "max_abs": 0.0067457873083185405, + "worst_column_rms": 0.0067457873083185405, + "worst_column": 7363, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "original_native": { + "rms": 0.00041784331596165115, + "max_abs": 0.0067457873083185405, + "worst_column_rms": 0.0067457873083185405, + "worst_column": 7363, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "mlx": { + "rms": 0.00041784331596204954, + "max_abs": 0.0067457873083185405, + "worst_column_rms": 0.0067457873083185405, + "worst_column": 7363, + "correctly_rounded": 7679, + "outputs": 7680 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "5d7265f9038c45a2a7414e9bf751a2cdee8ead2da6d0144ef8bc73500083f6f1" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-8bit", + "module": "language_model.model.layers.0.feed_forward.w1", + "input_stage": "layer_00_ffn_norm", + "input_file": "1.7B-8-ffn_gate.bf16", + "one_row_file": "1.7B-8-ffn_gate-one.bf16", + "model_directory": "LFM3-1.7B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 7680 + }, + { + "name": "1.7B-8-ffn_up", + "class": "ffn_up", + "shape": [ + 64, + 2048, + 7680 + ], + "input_sha256": "0ae00693590fcb04ffdf3d2b3b3e4f4781fc2040ba805edd75eb4fe43d02462e", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.0003413694321717928, + "max_abs": 0.006069664043025114, + "worst_column_rms": 0.0026501789267902973, + "worst_column": 3439, + "correctly_rounded": 491520, + "outputs": 491520 + }, + "original_native": { + "rms": 0.00034136943239456433, + "max_abs": 0.006069664043025114, + "worst_column_rms": 0.0026501789267902973, + "worst_column": 3439, + "correctly_rounded": 491474, + "outputs": 491520 + }, + "mlx": { + "rms": 0.00045176967752015936, + "max_abs": 0.006069664043025114, + "worst_column_rms": 0.002653870865120233, + "worst_column": 3439, + "correctly_rounded": 290023, + "outputs": 491520 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "6c9c205deb589766242f9176c5d5cb06b5f114ae4c0144c4b168c16fed380ace" + }, + { + "mode": "decode", + "native": { + "rms": 0.0003219362033343173, + "max_abs": 0.002960819299914874, + "worst_column_rms": 0.002960819299914874, + "worst_column": 1409, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "original_native": { + "rms": 0.0003219362033423213, + "max_abs": 0.002960819299914874, + "worst_column_rms": 0.002960819299914874, + "worst_column": 1409, + "correctly_rounded": 7679, + "outputs": 7680 + }, + "mlx": { + "rms": 0.0003219362033932306, + "max_abs": 0.002960819299914874, + "worst_column_rms": 0.002960819299914874, + "worst_column": 1409, + "correctly_rounded": 7679, + "outputs": 7680 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "44147335dcb02a2216a2d116959141df0a9aad2bec658fae007ae864ea0ec157" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-8bit", + "module": "language_model.model.layers.0.feed_forward.w3", + "input_stage": "layer_00_ffn_norm", + "input_file": "1.7B-8-ffn_up.bf16", + "one_row_file": "1.7B-8-ffn_up-one.bf16", + "model_directory": "LFM3-1.7B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 7680 + }, + { + "name": "1.7B-8-ffn_down", + "class": "ffn_down", + "shape": [ + 64, + 7680, + 2048 + ], + "input_sha256": "8c796e421c2f35fb094de23b40b44fb2741a45bc0361430791646f4e5047c374", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.0003090906799805102, + "max_abs": 0.022397026437175782, + "worst_column_rms": 0.006514728489646401, + "worst_column": 1402, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 0.0003090906800081439, + "max_abs": 0.022397026437175782, + "worst_column_rms": 0.006514728489646401, + "worst_column": 1402, + "correctly_rounded": 131048, + "outputs": 131072 + }, + "mlx": { + "rms": 0.00032086772346561653, + "max_abs": 0.022397026437175782, + "worst_column_rms": 0.006514728489646401, + "worst_column": 1402, + "correctly_rounded": 85661, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "c9535b39ba4ef83911282db1f28fca22373b318d7749ac18c90ca0a91a236bde" + }, + { + "mode": "decode", + "native": { + "rms": 9.402959689172091e-05, + "max_abs": 0.0014421518472715888, + "worst_column_rms": 0.0014421518472715888, + "worst_column": 1700, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 9.402959760299675e-05, + "max_abs": 0.0014421518472715888, + "worst_column_rms": 0.0014421518472715888, + "worst_column": 1700, + "correctly_rounded": 2047, + "outputs": 2048 + }, + "mlx": { + "rms": 9.402959760299675e-05, + "max_abs": 0.0014421518472715888, + "worst_column_rms": 0.0014421518472715888, + "worst_column": 1700, + "correctly_rounded": 2047, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "8555da8c544587c8042e0bf321c6288974d6f2e58eabf4079966ed53504e8510" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-8bit", + "module": "language_model.model.layers.0.feed_forward.w2", + "input_stage": "layer_00_act", + "input_file": "1.7B-8-ffn_down.bf16", + "one_row_file": "1.7B-8-ffn_down-one.bf16", + "model_directory": "LFM3-1.7B-MLX-8bit", + "rows": 64, + "k": 7680, + "n": 2048 + }, + { + "name": "1.7B-8-attention_q", + "class": "attention_q", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "9c6a7b9422c6719539de6b1e5f47868f46309de3229daa915e6ec928d63d0d6d", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.00012767604945446961, + "max_abs": 0.0019421786892053206, + "worst_column_rms": 0.0011111057008462014, + "worst_column": 1968, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 0.00012767604984798356, + "max_abs": 0.0019421786892053206, + "worst_column_rms": 0.0011111057008462014, + "worst_column": 1968, + "correctly_rounded": 131060, + "outputs": 131072 + }, + "mlx": { + "rms": 0.00014398357583066771, + "max_abs": 0.0022453422857324767, + "worst_column_rms": 0.001143927632495306, + "worst_column": 1968, + "correctly_rounded": 88228, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "cfeac51b5f537a6e2bcc4c466a9912a06d2b4c0d4d0e5418f04678bc99febb76" + }, + { + "mode": "decode", + "native": { + "rms": 6.831492422865153e-05, + "max_abs": 0.00047187499831125024, + "worst_column_rms": 0.00047187499831125024, + "worst_column": 487, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 6.831492422865153e-05, + "max_abs": 0.00047187499831125024, + "worst_column_rms": 0.00047187499831125024, + "worst_column": 487, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "mlx": { + "rms": 6.831492422865153e-05, + "max_abs": 0.00047187499831125024, + "worst_column_rms": 0.00047187499831125024, + "worst_column": 487, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "c12320e5ac155f07a393be2542fff7995eccd912097c79f8ac63aa3dadf750d0" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-8bit", + "module": "language_model.model.layers.2.self_attn.q_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "1.7B-8-attention_q.bf16", + "one_row_file": "1.7B-8-attention_q-one.bf16", + "model_directory": "LFM3-1.7B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 2048 + }, + { + "name": "1.7B-8-attention_k", + "class": "attention_k", + "shape": [ + 64, + 2048, + 512 + ], + "input_sha256": "9c6a7b9422c6719539de6b1e5f47868f46309de3229daa915e6ec928d63d0d6d", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.00013143387188008636, + "max_abs": 0.0018988696865562815, + "worst_column_rms": 0.0007003025948989976, + "worst_column": 158, + "correctly_rounded": 32768, + "outputs": 32768 + }, + "original_native": { + "rms": 0.00013143387188037554, + "max_abs": 0.0018988696865562815, + "worst_column_rms": 0.0007003025948989976, + "worst_column": 158, + "correctly_rounded": 32767, + "outputs": 32768 + }, + "mlx": { + "rms": 0.0001471129077504911, + "max_abs": 0.0018988696865562815, + "worst_column_rms": 0.0007154446095470521, + "worst_column": 158, + "correctly_rounded": 22763, + "outputs": 32768 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "0d64a33538106896db23021a7251d26e7622f90e950561d15f0d041b70ce4802" + }, + { + "mode": "decode", + "native": { + "rms": 9.240672747765843e-05, + "max_abs": 0.0008959722144936677, + "worst_column_rms": 0.0008959722144936677, + "worst_column": 103, + "correctly_rounded": 512, + "outputs": 512 + }, + "original_native": { + "rms": 9.240672747765843e-05, + "max_abs": 0.0008959722144936677, + "worst_column_rms": 0.0008959722144936677, + "worst_column": 103, + "correctly_rounded": 512, + "outputs": 512 + }, + "mlx": { + "rms": 9.240672747765843e-05, + "max_abs": 0.0008959722144936677, + "worst_column_rms": 0.0008959722144936677, + "worst_column": 103, + "correctly_rounded": 512, + "outputs": 512 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "bd2d6913ec4438c993dcebe577870b403caae745a38a7af3c9fe7ea2c849230e" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-8bit", + "module": "language_model.model.layers.2.self_attn.k_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "1.7B-8-attention_k.bf16", + "one_row_file": "1.7B-8-attention_k-one.bf16", + "model_directory": "LFM3-1.7B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 512 + }, + { + "name": "1.7B-8-attention_v", + "class": "attention_v", + "shape": [ + 64, + 2048, + 512 + ], + "input_sha256": "9c6a7b9422c6719539de6b1e5f47868f46309de3229daa915e6ec928d63d0d6d", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 9.951508287904088e-05, + "max_abs": 0.0017808252487157006, + "worst_column_rms": 0.0004852346492035805, + "worst_column": 87, + "correctly_rounded": 32768, + "outputs": 32768 + }, + "original_native": { + "rms": 9.951508299669427e-05, + "max_abs": 0.0017808252487157006, + "worst_column_rms": 0.0004852346492035805, + "worst_column": 87, + "correctly_rounded": 32761, + "outputs": 32768 + }, + "mlx": { + "rms": 0.0001192278403815664, + "max_abs": 0.0017808252487157006, + "worst_column_rms": 0.0004878790113707174, + "worst_column": 87, + "correctly_rounded": 20620, + "outputs": 32768 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "8bbf123e32dd083bd8515984bd3f5d24e8520e68366cf5cbff99f6c42f9e9bbd" + }, + { + "mode": "decode", + "native": { + "rms": 6.517696956171333e-05, + "max_abs": 0.0007998780802154215, + "worst_column_rms": 0.0007998780802154215, + "worst_column": 87, + "correctly_rounded": 512, + "outputs": 512 + }, + "original_native": { + "rms": 6.517696956171333e-05, + "max_abs": 0.0007998780802154215, + "worst_column_rms": 0.0007998780802154215, + "worst_column": 87, + "correctly_rounded": 512, + "outputs": 512 + }, + "mlx": { + "rms": 6.517696956171333e-05, + "max_abs": 0.0007998780802154215, + "worst_column_rms": 0.0007998780802154215, + "worst_column": 87, + "correctly_rounded": 512, + "outputs": 512 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "5672c89b1209a47f4db84bbbed85ad5677542e28f1f4bcdbc95098d2d1027aa0" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-8bit", + "module": "language_model.model.layers.2.self_attn.v_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "1.7B-8-attention_v.bf16", + "one_row_file": "1.7B-8-attention_v-one.bf16", + "model_directory": "LFM3-1.7B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 512 + }, + { + "name": "1.7B-8-attention_gate", + "class": "attention_gate", + "shape": [ + 64, + 2048, + 16 + ], + "input_sha256": "9c6a7b9422c6719539de6b1e5f47868f46309de3229daa915e6ec928d63d0d6d", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.0011212918759148215, + "max_abs": 0.00386981496012595, + "worst_column_rms": 0.0018269449275887483, + "worst_column": 13, + "correctly_rounded": 1024, + "outputs": 1024 + }, + "original_native": { + "rms": 0.0011212918759148215, + "max_abs": 0.00386981496012595, + "worst_column_rms": 0.0018269449275887483, + "worst_column": 13, + "correctly_rounded": 1024, + "outputs": 1024 + }, + "mlx": { + "rms": 0.0011661833427044997, + "max_abs": 0.004477753464016132, + "worst_column_rms": 0.0019055753286617715, + "worst_column": 13, + "correctly_rounded": 851, + "outputs": 1024 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "9d5e0d6dc77e56482a4652be9b53cdecfd809c91c6daf9cd86d598cf6ea5515d" + }, + { + "mode": "decode", + "native": { + "rms": 0.0004744856350952008, + "max_abs": 0.0014722358719154727, + "worst_column_rms": 0.0014722358719154727, + "worst_column": 13, + "correctly_rounded": 16, + "outputs": 16 + }, + "original_native": { + "rms": 0.0004744856350952008, + "max_abs": 0.0014722358719154727, + "worst_column_rms": 0.0014722358719154727, + "worst_column": 13, + "correctly_rounded": 16, + "outputs": 16 + }, + "mlx": { + "rms": 0.0004744856350952008, + "max_abs": 0.0014722358719154727, + "worst_column_rms": 0.0014722358719154727, + "worst_column": 13, + "correctly_rounded": 16, + "outputs": 16 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "490faa9e36e8cd958d4d64e8f85a708191d50e1c8b402c5686e8bdeefa97d7ae" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-8bit", + "module": "language_model.model.layers.2.self_attn.gate_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "1.7B-8-attention_gate.bf16", + "one_row_file": "1.7B-8-attention_gate-one.bf16", + "model_directory": "LFM3-1.7B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 16 + }, + { + "name": "1.7B-8-attention_out", + "class": "attention_out", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "ace6b8df600a44193c8f0e7d9bcd1e1e9d68f6d0ad6946f629148b135a62b0a1", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 3.48718231466456e-05, + "max_abs": 0.0009649635296682391, + "worst_column_rms": 0.00046214981494814637, + "worst_column": 1777, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 3.487182315325104e-05, + "max_abs": 0.0009649635296682391, + "worst_column_rms": 0.00046214981494814637, + "worst_column": 1777, + "correctly_rounded": 131058, + "outputs": 131072 + }, + "mlx": { + "rms": 4.234818981602883e-05, + "max_abs": 0.0011106174529231794, + "worst_column_rms": 0.0004875995779470689, + "worst_column": 1777, + "correctly_rounded": 77604, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "9309a2af2e0a4d428c0c5dc0cea2732f468c93aaff21a7de62689c7c949b445f" + }, + { + "mode": "decode", + "native": { + "rms": 3.123719442125664e-05, + "max_abs": 0.00047937290219124407, + "worst_column_rms": 0.00047937290219124407, + "worst_column": 884, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 3.1237194421293076e-05, + "max_abs": 0.00047937290219124407, + "worst_column_rms": 0.00047937290219124407, + "worst_column": 884, + "correctly_rounded": 2047, + "outputs": 2048 + }, + "mlx": { + "rms": 3.123719442125664e-05, + "max_abs": 0.00047937290219124407, + "worst_column_rms": 0.00047937290219124407, + "worst_column": 884, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "9aee14bdf4e46e44048a96d2ecce55556f03067b17f970bb6b693c6d3c6cb9fd" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-8bit", + "module": "language_model.model.layers.2.self_attn.o_proj", + "input_stage": "layer_02_attn_gated", + "input_file": "1.7B-8-attention_out.bf16", + "one_row_file": "1.7B-8-attention_out-one.bf16", + "model_directory": "LFM3-1.7B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 2048 + }, + { + "name": "1.7B-8-vocabulary_head", + "class": "vocabulary_head", + "shape": [ + 64, + 2048, + 128000 + ], + "input_sha256": "f49abdc5af55f8e615eaae768a124bcbae464a7bea59a64799e085fdf4e07e15", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.004740064136573247, + "max_abs": 0.062337145340279676, + "worst_column_rms": 0.027917632414962472, + "worst_column": 278, + "correctly_rounded": 8192000, + "outputs": 8192000 + }, + "original_native": { + "rms": 0.0047400641414997976, + "max_abs": 0.062337145340279676, + "worst_column_rms": 0.027917632414962472, + "worst_column": 278, + "correctly_rounded": 8191302, + "outputs": 8192000 + }, + "mlx": { + "rms": 0.005486790466557924, + "max_abs": 0.06816060113487765, + "worst_column_rms": 0.028007390548458203, + "worst_column": 278, + "correctly_rounded": 5592458, + "outputs": 8192000 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "60e65041ceddf857ed5d3f834ee2e3e9031aa35657546522c649c229cc69cadd" + }, + { + "mode": "decode", + "native": { + "rms": 0.004022961455732135, + "max_abs": 0.03119968496321235, + "worst_column_rms": 0.03119968496321235, + "worst_column": 395, + "correctly_rounded": 128000, + "outputs": 128000 + }, + "original_native": { + "rms": 0.004022961457516284, + "max_abs": 0.03119968496321235, + "worst_column_rms": 0.03119968496321235, + "worst_column": 395, + "correctly_rounded": 127992, + "outputs": 128000 + }, + "mlx": { + "rms": 0.004022961457520553, + "max_abs": 0.03119968496321235, + "worst_column_rms": 0.03119968496321235, + "worst_column": 395, + "correctly_rounded": 127993, + "outputs": 128000 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "0eb8a7d5cff089e99330ff2b8de752bcbde52172e7a80d47f97553714027587c" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-1.7B-MLX-8bit", + "module": "language_model.model.embed_tokens", + "input_stage": "final_norm", + "input_file": "1.7B-8-vocabulary_head.bf16", + "one_row_file": "1.7B-8-vocabulary_head-one.bf16", + "model_directory": "LFM3-1.7B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 128000 + }, + { + "name": "VL-2B-4-conv_in", + "class": "conv_in", + "shape": [ + 64, + 2048, + 6144 + ], + "input_sha256": "ccdca3e9d818ce2ef8822cee5071ac88761b956757ed9f2db317cd896708cc6e", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.000301678319257443, + "max_abs": 0.0044559728557942435, + "worst_column_rms": 0.0013578813383313725, + "worst_column": 2170, + "correctly_rounded": 393216, + "outputs": 393216 + }, + "original_native": { + "rms": 0.0003016783193256304, + "max_abs": 0.0044559728557942435, + "worst_column_rms": 0.0013578813383313725, + "worst_column": 2170, + "correctly_rounded": 393172, + "outputs": 393216 + }, + "mlx": { + "rms": 0.0003612506309635582, + "max_abs": 0.0044559728557942435, + "worst_column_rms": 0.0013637130326949528, + "worst_column": 2170, + "correctly_rounded": 241077, + "outputs": 393216 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "d14b70a308f20d5218b334611a93b842b44e8608c806f4e7e7fffc0ad14a9e9f" + }, + { + "mode": "decode", + "native": { + "rms": 0.0003519487431762353, + "max_abs": 0.0019513567458488978, + "worst_column_rms": 0.0019513567458488978, + "worst_column": 593, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "original_native": { + "rms": 0.0003519487431762353, + "max_abs": 0.0019513567458488978, + "worst_column_rms": 0.0019513567458488978, + "worst_column": 593, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "mlx": { + "rms": 0.0003519487431762353, + "max_abs": 0.0019513567458488978, + "worst_column_rms": 0.0019513567458488978, + "worst_column": 593, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "719399211617079e28556f60b5da3216dc841a0c5f40f51223f210e081c14003" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-4bit", + "module": "language_model.model.layers.0.conv.in_proj", + "input_stage": "layer_00_operator_norm", + "input_file": "VL-2B-4-conv_in.bf16", + "one_row_file": "VL-2B-4-conv_in-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 6144 + }, + { + "name": "VL-2B-4-conv_out", + "class": "conv_out", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "c25e5dd7f66e6c0c6a83b2f86d96d7940c1cb250a83685d5a03250b69dfa11bb", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 5.921648485619485e-06, + "max_abs": 0.00024387097609235298, + "worst_column_rms": 0.00013768757463218447, + "worst_column": 1688, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 5.921648488106127e-06, + "max_abs": 0.00024387097609235298, + "worst_column_rms": 0.00013768757463218447, + "worst_column": 1688, + "correctly_rounded": 131057, + "outputs": 131072 + }, + "mlx": { + "rms": 6.101400915324444e-06, + "max_abs": 0.000244410273907647, + "worst_column_rms": 0.0001379519287160541, + "worst_column": 1688, + "correctly_rounded": 92741, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "a781c7e21bd0ed2ed093e11d769fa51868182e8a96a8ccaf8006527cf768a1cb" + }, + { + "mode": "decode", + "native": { + "rms": 1.4914469420485761e-06, + "max_abs": 3.3471219616165726e-05, + "worst_column_rms": 3.3471219616165726e-05, + "worst_column": 42, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 1.4914470644961778e-06, + "max_abs": 3.3471219616165726e-05, + "worst_column_rms": 3.3471219616165726e-05, + "worst_column": 42, + "correctly_rounded": 2047, + "outputs": 2048 + }, + "mlx": { + "rms": 1.4914470644961778e-06, + "max_abs": 3.3471219616165726e-05, + "worst_column_rms": 3.3471219616165726e-05, + "worst_column": 42, + "correctly_rounded": 2047, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "8bc576734bd022af17e8c1d5f6e0e38088c42165aa06e297822024a3762dd806" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-4bit", + "module": "language_model.model.layers.0.conv.out_proj", + "input_stage": "layer_00_conv_gated", + "input_file": "VL-2B-4-conv_out.bf16", + "one_row_file": "VL-2B-4-conv_out-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 2048 + }, + { + "name": "VL-2B-4-ffn_gate", + "class": "ffn_gate", + "shape": [ + 64, + 2048, + 7680 + ], + "input_sha256": "51c210a502fafe12985bf1a9e52b4addff6d16fb026e0143e0325854e552f61e", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.0002935762767977053, + "max_abs": 0.0038836358580738306, + "worst_column_rms": 0.002337548940558138, + "worst_column": 445, + "correctly_rounded": 491520, + "outputs": 491520 + }, + "original_native": { + "rms": 0.0002935762769197033, + "max_abs": 0.0038836358580738306, + "worst_column_rms": 0.002337548940558138, + "worst_column": 445, + "correctly_rounded": 491494, + "outputs": 491520 + }, + "mlx": { + "rms": 0.00032621324435281294, + "max_abs": 0.00396678494871594, + "worst_column_rms": 0.002343862865001316, + "worst_column": 445, + "correctly_rounded": 363182, + "outputs": 491520 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "4f23d79f8e70a5da00d1b6c7369456881ce93f202bd7677c731a42b8bf13afa7" + }, + { + "mode": "decode", + "native": { + "rms": 0.0001697117900273488, + "max_abs": 0.0011569455091375858, + "worst_column_rms": 0.0011569455091375858, + "worst_column": 494, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "original_native": { + "rms": 0.0001697117900273488, + "max_abs": 0.0011569455091375858, + "worst_column_rms": 0.0011569455091375858, + "worst_column": 494, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "mlx": { + "rms": 0.0001697117900273488, + "max_abs": 0.0011569455091375858, + "worst_column_rms": 0.0011569455091375858, + "worst_column": 494, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "fc3260f84dd8a6cd21baa4e4690c37e26e2a09b75f27bdccfc0b48e1a4e155a2" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-4bit", + "module": "language_model.model.layers.0.feed_forward.w1", + "input_stage": "layer_00_ffn_norm", + "input_file": "VL-2B-4-ffn_gate.bf16", + "one_row_file": "VL-2B-4-ffn_gate-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 7680 + }, + { + "name": "VL-2B-4-ffn_up", + "class": "ffn_up", + "shape": [ + 64, + 2048, + 7680 + ], + "input_sha256": "51c210a502fafe12985bf1a9e52b4addff6d16fb026e0143e0325854e552f61e", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.00017417406370842336, + "max_abs": 0.0019354705000296235, + "worst_column_rms": 0.0006451826873528963, + "worst_column": 445, + "correctly_rounded": 491520, + "outputs": 491520 + }, + "original_native": { + "rms": 0.00017417406373572864, + "max_abs": 0.0019354705000296235, + "worst_column_rms": 0.0006451826873528963, + "worst_column": 445, + "correctly_rounded": 491482, + "outputs": 491520 + }, + "mlx": { + "rms": 0.00021669173400815016, + "max_abs": 0.0019707794999703765, + "worst_column_rms": 0.0006483897553762815, + "worst_column": 445, + "correctly_rounded": 310825, + "outputs": 491520 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "50a86965edc35a4281b42ad4574c34de38cf90ae70309a694c54f6fc9033b813" + }, + { + "mode": "decode", + "native": { + "rms": 0.00015682482467182469, + "max_abs": 0.0013326654152479023, + "worst_column_rms": 0.0013326654152479023, + "worst_column": 1709, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "original_native": { + "rms": 0.00015682482467182469, + "max_abs": 0.0013326654152479023, + "worst_column_rms": 0.0013326654152479023, + "worst_column": 1709, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "mlx": { + "rms": 0.00015682482467182469, + "max_abs": 0.0013326654152479023, + "worst_column_rms": 0.0013326654152479023, + "worst_column": 1709, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "14890b0147213d78a0d31c91dbdb82efbef19a820cc80d281b37d0441d2963b5" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-4bit", + "module": "language_model.model.layers.0.feed_forward.w3", + "input_stage": "layer_00_ffn_norm", + "input_file": "VL-2B-4-ffn_up.bf16", + "one_row_file": "VL-2B-4-ffn_up-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 7680 + }, + { + "name": "VL-2B-4-ffn_down", + "class": "ffn_down", + "shape": [ + 64, + 7680, + 2048 + ], + "input_sha256": "32f733a66f8111e639b61c8312c890a40efb88adc651cb08984454238308f5b6", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 7.921382457934971e-05, + "max_abs": 0.0038283297399175353, + "worst_column_rms": 0.0015575129469822303, + "worst_column": 1700, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 7.921382457946553e-05, + "max_abs": 0.0038283297399175353, + "worst_column_rms": 0.0015575129469822303, + "worst_column": 1700, + "correctly_rounded": 131056, + "outputs": 131072 + }, + "mlx": { + "rms": 8.159535860245026e-05, + "max_abs": 0.003984170260082465, + "worst_column_rms": 0.0015703229165001632, + "worst_column": 1700, + "correctly_rounded": 93018, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "10283fdeee7e16d060a6e045bb7f91c25dadc1cdc0251176a658a9b4c2126733" + }, + { + "mode": "decode", + "native": { + "rms": 2.496907836733995e-05, + "max_abs": 0.00045843876421258756, + "worst_column_rms": 0.00045843876421258756, + "worst_column": 1688, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 2.496907836733995e-05, + "max_abs": 0.00045843876421258756, + "worst_column_rms": 0.00045843876421258756, + "worst_column": 1688, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "mlx": { + "rms": 2.496907836733995e-05, + "max_abs": 0.00045843876421258756, + "worst_column_rms": 0.00045843876421258756, + "worst_column": 1688, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "98c725a270344e983bedaeaba0de68165a5ff08404ca24012c943835d7a41b4d" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-4bit", + "module": "language_model.model.layers.0.feed_forward.w2", + "input_stage": "layer_00_act", + "input_file": "VL-2B-4-ffn_down.bf16", + "one_row_file": "VL-2B-4-ffn_down-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-4bit", + "rows": 64, + "k": 7680, + "n": 2048 + }, + { + "name": "VL-2B-4-attention_q", + "class": "attention_q", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "ac1a507d9a12b4da7985b104bb8518f5b8642c79ebb85920717e72cb503d7399", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 9.203033963530209e-05, + "max_abs": 0.0017881895473692566, + "worst_column_rms": 0.0006346625347828979, + "worst_column": 1968, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 9.203034019676422e-05, + "max_abs": 0.0017881895473692566, + "worst_column_rms": 0.0006346625347828979, + "worst_column": 1968, + "correctly_rounded": 131065, + "outputs": 131072 + }, + "mlx": { + "rms": 9.864548573794255e-05, + "max_abs": 0.0017881895473692566, + "worst_column_rms": 0.0006431931735118539, + "worst_column": 38, + "correctly_rounded": 94833, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "ccdfc7269632f8e52d30bbc34c55535b7fce5f67ad555f75a4b40f4ed77ebf03" + }, + { + "mode": "decode", + "native": { + "rms": 6.454564954660002e-05, + "max_abs": 0.00046773780923103914, + "worst_column_rms": 0.00046773780923103914, + "worst_column": 735, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 6.454564954660002e-05, + "max_abs": 0.00046773780923103914, + "worst_column_rms": 0.00046773780923103914, + "worst_column": 735, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "mlx": { + "rms": 6.454564954660002e-05, + "max_abs": 0.00046773780923103914, + "worst_column_rms": 0.00046773780923103914, + "worst_column": 735, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "80c2bb98cc9414ddeb8708afe894f12a1c78380124bcf65b261af28d5afba456" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-4bit", + "module": "language_model.model.layers.2.self_attn.q_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "VL-2B-4-attention_q.bf16", + "one_row_file": "VL-2B-4-attention_q-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 2048 + }, + { + "name": "VL-2B-4-attention_k", + "class": "attention_k", + "shape": [ + 64, + 2048, + 512 + ], + "input_sha256": "ac1a507d9a12b4da7985b104bb8518f5b8642c79ebb85920717e72cb503d7399", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 8.884895694585604e-05, + "max_abs": 0.0009643462399253622, + "worst_column_rms": 0.0003652284274346588, + "worst_column": 158, + "correctly_rounded": 32768, + "outputs": 32768 + }, + "original_native": { + "rms": 8.884895694637085e-05, + "max_abs": 0.0009643462399253622, + "worst_column_rms": 0.0003652284274346588, + "worst_column": 158, + "correctly_rounded": 32765, + "outputs": 32768 + }, + "mlx": { + "rms": 9.49308760841846e-05, + "max_abs": 0.0009887787600746378, + "worst_column_rms": 0.0003672642739108803, + "worst_column": 158, + "correctly_rounded": 24429, + "outputs": 32768 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "6c46ee9e09a66308ad24f7fcc48c806f5066f11b77cad9612cd356ffe4634728" + }, + { + "mode": "decode", + "native": { + "rms": 7.471209415503583e-05, + "max_abs": 0.0004414465875015594, + "worst_column_rms": 0.0004414465875015594, + "worst_column": 158, + "correctly_rounded": 512, + "outputs": 512 + }, + "original_native": { + "rms": 7.471209415503583e-05, + "max_abs": 0.0004414465875015594, + "worst_column_rms": 0.0004414465875015594, + "worst_column": 158, + "correctly_rounded": 512, + "outputs": 512 + }, + "mlx": { + "rms": 7.471209415503583e-05, + "max_abs": 0.0004414465875015594, + "worst_column_rms": 0.0004414465875015594, + "worst_column": 158, + "correctly_rounded": 512, + "outputs": 512 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "1b51d5ed6ef29f58baa2788eafaa855f89bb12320babdc361f2cd9f58816ac5b" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-4bit", + "module": "language_model.model.layers.2.self_attn.k_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "VL-2B-4-attention_k.bf16", + "one_row_file": "VL-2B-4-attention_k-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 512 + }, + { + "name": "VL-2B-4-attention_v", + "class": "attention_v", + "shape": [ + 64, + 2048, + 512 + ], + "input_sha256": "ac1a507d9a12b4da7985b104bb8518f5b8642c79ebb85920717e72cb503d7399", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 7.22027634012478e-05, + "max_abs": 0.0009573189599905163, + "worst_column_rms": 0.00021734058567212002, + "worst_column": 87, + "correctly_rounded": 32768, + "outputs": 32768 + }, + "original_native": { + "rms": 7.22027634083276e-05, + "max_abs": 0.0009573189599905163, + "worst_column_rms": 0.00021734058567212002, + "worst_column": 87, + "correctly_rounded": 32759, + "outputs": 32768 + }, + "mlx": { + "rms": 8.130238068851228e-05, + "max_abs": 0.0009958060400094837, + "worst_column_rms": 0.00022184840738541725, + "worst_column": 87, + "correctly_rounded": 22925, + "outputs": 32768 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "b36ddb06295061232d83d792cf1d992f88aff3c9394bb745575bfa07e66ffdb4" + }, + { + "mode": "decode", + "native": { + "rms": 7.877108547938019e-05, + "max_abs": 0.00045827232315787114, + "worst_column_rms": 0.00045827232315787114, + "worst_column": 361, + "correctly_rounded": 512, + "outputs": 512 + }, + "original_native": { + "rms": 7.877108547938019e-05, + "max_abs": 0.00045827232315787114, + "worst_column_rms": 0.00045827232315787114, + "worst_column": 361, + "correctly_rounded": 512, + "outputs": 512 + }, + "mlx": { + "rms": 7.877108547938019e-05, + "max_abs": 0.00045827232315787114, + "worst_column_rms": 0.00045827232315787114, + "worst_column": 361, + "correctly_rounded": 512, + "outputs": 512 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "e46f37723d8adf961c8e8e62cb1d09d74a33fdd36562fb52bb7f804aeccb129c" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-4bit", + "module": "language_model.model.layers.2.self_attn.v_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "VL-2B-4-attention_v.bf16", + "one_row_file": "VL-2B-4-attention_v-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 512 + }, + { + "name": "VL-2B-4-attention_gate", + "class": "attention_gate", + "shape": [ + 64, + 2048, + 16 + ], + "input_sha256": "ac1a507d9a12b4da7985b104bb8518f5b8642c79ebb85920717e72cb503d7399", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.0013228336389065115, + "max_abs": 0.0038743025506846607, + "worst_column_rms": 0.0020973974602657747, + "worst_column": 7, + "correctly_rounded": 1024, + "outputs": 1024 + }, + "original_native": { + "rms": 0.0013228336389065115, + "max_abs": 0.0038743025506846607, + "worst_column_rms": 0.0020973974602657747, + "worst_column": 7, + "correctly_rounded": 1024, + "outputs": 1024 + }, + "mlx": { + "rms": 0.0013396723066382379, + "max_abs": 0.004102444683667272, + "worst_column_rms": 0.0021203127030923895, + "worst_column": 7, + "correctly_rounded": 955, + "outputs": 1024 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "a18b3d969a2b0b1ded99fc9bdf947a780f5c8b711ea1c57a893bb2d9f2dcb810" + }, + { + "mode": "decode", + "native": { + "rms": 0.0007187758730734035, + "max_abs": 0.0012749253655783832, + "worst_column_rms": 0.0012749253655783832, + "worst_column": 13, + "correctly_rounded": 16, + "outputs": 16 + }, + "original_native": { + "rms": 0.0007187758730734035, + "max_abs": 0.0012749253655783832, + "worst_column_rms": 0.0012749253655783832, + "worst_column": 13, + "correctly_rounded": 16, + "outputs": 16 + }, + "mlx": { + "rms": 0.0007187758730734035, + "max_abs": 0.0012749253655783832, + "worst_column_rms": 0.0012749253655783832, + "worst_column": 13, + "correctly_rounded": 16, + "outputs": 16 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "3137099219d5c4ecf39915438ad8f89e506b5b84b2898fa3dcd4b0306f33f77d" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-4bit", + "module": "language_model.model.layers.2.self_attn.gate_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "VL-2B-4-attention_gate.bf16", + "one_row_file": "VL-2B-4-attention_gate-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 16 + }, + { + "name": "VL-2B-4-attention_out", + "class": "attention_out", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "6e5bc5e7c28cb5fa899ce1a3a6525b6a2c69282f0adcff2cac1cdd47a89ec47f", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 1.5764546724247333e-05, + "max_abs": 0.0004781260845447832, + "worst_column_rms": 0.00016358814062099658, + "worst_column": 1918, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 1.5764546736146177e-05, + "max_abs": 0.0004781260845447832, + "worst_column_rms": 0.00016358814062099658, + "worst_column": 1918, + "correctly_rounded": 131054, + "outputs": 131072 + }, + "mlx": { + "rms": 1.8953483640096274e-05, + "max_abs": 0.0004984364154552168, + "worst_column_rms": 0.00016458914888542233, + "worst_column": 1918, + "correctly_rounded": 81822, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "193973f3a4fc797a29993f8a8c52b3659f13a2a044740cc8061011a9e120bcf8" + }, + { + "mode": "decode", + "native": { + "rms": 2.679517213492092e-05, + "max_abs": 0.00033973245444940403, + "worst_column_rms": 0.00033973245444940403, + "worst_column": 1996, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 2.6795172169693463e-05, + "max_abs": 0.00033973245444940403, + "worst_column_rms": 0.00033973245444940403, + "worst_column": 1996, + "correctly_rounded": 2047, + "outputs": 2048 + }, + "mlx": { + "rms": 2.6795172169693463e-05, + "max_abs": 0.00033973245444940403, + "worst_column_rms": 0.00033973245444940403, + "worst_column": 1996, + "correctly_rounded": 2047, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "2f9d4462c35d055eb24a15602a150770bd2387229cdd4ae7a028b26a4dc86927" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-4bit", + "module": "language_model.model.layers.2.self_attn.o_proj", + "input_stage": "layer_02_attn_gated", + "input_file": "VL-2B-4-attention_out.bf16", + "one_row_file": "VL-2B-4-attention_out-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 2048 + }, + { + "name": "VL-2B-4-vocabulary_head", + "class": "vocabulary_head", + "shape": [ + 64, + 2048, + 128000 + ], + "input_sha256": "91bc4c54d9772e05164e553987fd3441c058e627c92ff54c805243ded1eb1055", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.0066628096956532455, + "max_abs": 0.05923806325881742, + "worst_column_rms": 0.016799600907355324, + "worst_column": 278, + "correctly_rounded": 8192000, + "outputs": 8192000 + }, + "original_native": { + "rms": 0.006662809704016998, + "max_abs": 0.05923806325881742, + "worst_column_rms": 0.016799600907355324, + "worst_column": 278, + "correctly_rounded": 8191610, + "outputs": 8192000 + }, + "mlx": { + "rms": 0.00750207421815752, + "max_abs": 0.05923806325881742, + "worst_column_rms": 0.017060511128459417, + "worst_column": 278, + "correctly_rounded": 6517082, + "outputs": 8192000 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "64bd27b14f896401645b37e539c37ae3d35d28c1df24bc48f6a4c18fabf78a4c" + }, + { + "mode": "decode", + "native": { + "rms": 0.005822228238020538, + "max_abs": 0.029901276400778443, + "worst_column_rms": 0.029901276400778443, + "worst_column": 20, + "correctly_rounded": 128000, + "outputs": 128000 + }, + "original_native": { + "rms": 0.005822228238038583, + "max_abs": 0.029901276400778443, + "worst_column_rms": 0.029901276400778443, + "worst_column": 20, + "correctly_rounded": 127996, + "outputs": 128000 + }, + "mlx": { + "rms": 0.005822228238128246, + "max_abs": 0.029901276400778443, + "worst_column_rms": 0.029901276400778443, + "worst_column": 20, + "correctly_rounded": 127995, + "outputs": 128000 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "95b91cd0681074c14320eaa82430d3ddd9d21a5015e726c98c797b96206ffd0e" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-4bit", + "module": "language_model.model.embed_tokens", + "input_stage": "final_norm", + "input_file": "VL-2B-4-vocabulary_head.bf16", + "one_row_file": "VL-2B-4-vocabulary_head-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-4bit", + "rows": 64, + "k": 2048, + "n": 128000 + }, + { + "name": "VL-2B-6-conv_in", + "class": "conv_in", + "shape": [ + 64, + 2048, + 6144 + ], + "input_sha256": "22687fea18a3550d080bdf2e9b38d394b1371faba7ed0888256ca3a6dacce6c5", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.0003008418803730303, + "max_abs": 0.0038474345055874437, + "worst_column_rms": 0.0012868583045841907, + "worst_column": 5772, + "correctly_rounded": 393216, + "outputs": 393216 + }, + "original_native": { + "rms": 0.0003008418804129762, + "max_abs": 0.0038474345055874437, + "worst_column_rms": 0.0012868583045841907, + "worst_column": 5772, + "correctly_rounded": 393170, + "outputs": 393216 + }, + "mlx": { + "rms": 0.0003834423066953888, + "max_abs": 0.003965065494412556, + "worst_column_rms": 0.0013497252024361544, + "worst_column": 2498, + "correctly_rounded": 225087, + "outputs": 393216 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "767f316b194381f15ad6ecff5a88bb2764cf709d4e1ef2d841e32a52cc960ad3" + }, + { + "mode": "decode", + "native": { + "rms": 0.00034729702258979624, + "max_abs": 0.0019271294913778547, + "worst_column_rms": 0.0019271294913778547, + "worst_column": 953, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "original_native": { + "rms": 0.00034729702258979624, + "max_abs": 0.0019271294913778547, + "worst_column_rms": 0.0019271294913778547, + "worst_column": 953, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "mlx": { + "rms": 0.000347297022589876, + "max_abs": 0.0019271294913778547, + "worst_column_rms": 0.0019271294913778547, + "worst_column": 953, + "correctly_rounded": 6143, + "outputs": 6144 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "f747092d8186ca673cd682c56a429eaf2c27631c02b9903ba803532865043db5" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-6bit", + "module": "language_model.model.layers.0.conv.in_proj", + "input_stage": "layer_00_operator_norm", + "input_file": "VL-2B-6-conv_in.bf16", + "one_row_file": "VL-2B-6-conv_in-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 6144 + }, + { + "name": "VL-2B-6-conv_out", + "class": "conv_out", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "f71940f01c70c561ab4a511d35fd6311b06fe7f1a13737124eb85dca25c1c059", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 7.008236644171265e-06, + "max_abs": 0.0004590110785187562, + "worst_column_rms": 0.00017687126198040487, + "worst_column": 1688, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 7.008236644263026e-06, + "max_abs": 0.0004590110785187562, + "worst_column_rms": 0.00017687126198040487, + "worst_column": 1688, + "correctly_rounded": 131055, + "outputs": 131072 + }, + "mlx": { + "rms": 7.349558052858887e-06, + "max_abs": 0.0004590110785187562, + "worst_column_rms": 0.00018377644416154546, + "worst_column": 1688, + "correctly_rounded": 85884, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "e133c7b157ae2341d7c533a153789ed4c8ff5fe93df6146d57cf855ee81f5d93" + }, + { + "mode": "decode", + "native": { + "rms": 1.8153997898761082e-06, + "max_abs": 5.941687102972715e-05, + "worst_column_rms": 5.941687102972715e-05, + "worst_column": 42, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 1.815399797246497e-06, + "max_abs": 5.941687102972715e-05, + "worst_column_rms": 5.941687102972715e-05, + "worst_column": 42, + "correctly_rounded": 2047, + "outputs": 2048 + }, + "mlx": { + "rms": 1.8153997898761082e-06, + "max_abs": 5.941687102972715e-05, + "worst_column_rms": 5.941687102972715e-05, + "worst_column": 42, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "23087af72979d3a2dc227423c1b8c31d4ff5954224f07798b82dd730c494570a" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-6bit", + "module": "language_model.model.layers.0.conv.out_proj", + "input_stage": "layer_00_conv_gated", + "input_file": "VL-2B-6-conv_out.bf16", + "one_row_file": "VL-2B-6-conv_out-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 2048 + }, + { + "name": "VL-2B-6-ffn_gate", + "class": "ffn_gate", + "shape": [ + 64, + 2048, + 7680 + ], + "input_sha256": "44917aad17a99e1c011e66b1d0175376613dcf92e383659f09b60abf93d16755", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.0002935140869167439, + "max_abs": 0.003905432822648436, + "worst_column_rms": 0.0023571397925776835, + "worst_column": 6325, + "correctly_rounded": 491520, + "outputs": 491520 + }, + "original_native": { + "rms": 0.0002935140869965826, + "max_abs": 0.003905432822648436, + "worst_column_rms": 0.0023571397925776835, + "worst_column": 6325, + "correctly_rounded": 491486, + "outputs": 491520 + }, + "mlx": { + "rms": 0.000340048864451867, + "max_abs": 0.004516652450547554, + "worst_column_rms": 0.0025044884496532895, + "worst_column": 6325, + "correctly_rounded": 346174, + "outputs": 491520 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "5e22fb9cad2fbf1de6f75a4bd152691c72fec6c5daffab2da265ed56e0d31fcd" + }, + { + "mode": "decode", + "native": { + "rms": 0.00017282650030867917, + "max_abs": 0.0018593218212572538, + "worst_column_rms": 0.0018593218212572538, + "worst_column": 494, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "original_native": { + "rms": 0.000172826500338854, + "max_abs": 0.0018593218212572538, + "worst_column_rms": 0.0018593218212572538, + "worst_column": 494, + "correctly_rounded": 7679, + "outputs": 7680 + }, + "mlx": { + "rms": 0.00017282650033889843, + "max_abs": 0.0018593218212572538, + "worst_column_rms": 0.0018593218212572538, + "worst_column": 494, + "correctly_rounded": 7678, + "outputs": 7680 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "2d240b08a3628383264bb75eb0b522693059025db8cadb554d77b493703d0e9f" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-6bit", + "module": "language_model.model.layers.0.feed_forward.w1", + "input_stage": "layer_00_ffn_norm", + "input_file": "VL-2B-6-ffn_gate.bf16", + "one_row_file": "VL-2B-6-ffn_gate-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 7680 + }, + { + "name": "VL-2B-6-ffn_up", + "class": "ffn_up", + "shape": [ + 64, + 2048, + 7680 + ], + "input_sha256": "44917aad17a99e1c011e66b1d0175376613dcf92e383659f09b60abf93d16755", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.00017493355828698655, + "max_abs": 0.003105565559962997, + "worst_column_rms": 0.0006903703438797418, + "worst_column": 5656, + "correctly_rounded": 491520, + "outputs": 491520 + }, + "original_native": { + "rms": 0.000174933558530465, + "max_abs": 0.003105565559962997, + "worst_column_rms": 0.0006903703438797418, + "worst_column": 5656, + "correctly_rounded": 491485, + "outputs": 491520 + }, + "mlx": { + "rms": 0.00023395871241519814, + "max_abs": 0.003105565559962997, + "worst_column_rms": 0.0007007647839905683, + "worst_column": 5656, + "correctly_rounded": 289927, + "outputs": 491520 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "db0fa182e03793b1f02d4b297035039cf6af4001d0bb3f70cef2674b15eb4270" + }, + { + "mode": "decode", + "native": { + "rms": 0.0001548746080953618, + "max_abs": 0.0009255550778561883, + "worst_column_rms": 0.0009255550778561883, + "worst_column": 225, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "original_native": { + "rms": 0.00015487460846797592, + "max_abs": 0.0009255550778561883, + "worst_column_rms": 0.0009255550778561883, + "worst_column": 225, + "correctly_rounded": 7679, + "outputs": 7680 + }, + "mlx": { + "rms": 0.0001548746080953618, + "max_abs": 0.0009255550778561883, + "worst_column_rms": 0.0009255550778561883, + "worst_column": 225, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "4ff805d85a0816552f0774749cd18aa1daac3e67e74345b907331358fca800a8" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-6bit", + "module": "language_model.model.layers.0.feed_forward.w3", + "input_stage": "layer_00_ffn_norm", + "input_file": "VL-2B-6-ffn_up.bf16", + "one_row_file": "VL-2B-6-ffn_up-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 7680 + }, + { + "name": "VL-2B-6-ffn_down", + "class": "ffn_down", + "shape": [ + 64, + 7680, + 2048 + ], + "input_sha256": "962f75129b292bc0507b4416449b03d8db52f001459d58271bde708fcf0e3028", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 7.93460092904979e-05, + "max_abs": 0.003844046572638149, + "worst_column_rms": 0.0015947685186338543, + "worst_column": 1402, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 7.93460092964076e-05, + "max_abs": 0.003844046572638149, + "worst_column_rms": 0.0015947685186338543, + "worst_column": 1402, + "correctly_rounded": 131048, + "outputs": 131072 + }, + "mlx": { + "rms": 8.273406408620398e-05, + "max_abs": 0.003844046572638149, + "worst_column_rms": 0.001599518599316139, + "worst_column": 1402, + "correctly_rounded": 87561, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "2daecfa2bf09af4963cbc61d1fc1cd8c187d12b7f94ad5e3faa7bbdfcf164c52" + }, + { + "mode": "decode", + "native": { + "rms": 2.6558245120969697e-05, + "max_abs": 0.00046149368262149437, + "worst_column_rms": 0.00046149368262149437, + "worst_column": 1402, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 2.6558245120969697e-05, + "max_abs": 0.00046149368262149437, + "worst_column_rms": 0.00046149368262149437, + "worst_column": 1402, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "mlx": { + "rms": 2.655824512154325e-05, + "max_abs": 0.00046149368262149437, + "worst_column_rms": 0.00046149368262149437, + "worst_column": 1402, + "correctly_rounded": 2047, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "531988206cfc6f52e7e6b3c227132392bf3204b87a84f422bcd5bb80c162d68a" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-6bit", + "module": "language_model.model.layers.0.feed_forward.w2", + "input_stage": "layer_00_act", + "input_file": "VL-2B-6-ffn_down.bf16", + "one_row_file": "VL-2B-6-ffn_down-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-6bit", + "rows": 64, + "k": 7680, + "n": 2048 + }, + { + "name": "VL-2B-6-attention_q", + "class": "attention_q", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "35ec818b558d14a1b2243b8760ed0fd8bda97895272ea6ced3cc5ae6dcb5f0c9", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 9.082362206708874e-05, + "max_abs": 0.0019119152639177628, + "worst_column_rms": 0.0006550130820460575, + "worst_column": 1968, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 9.082362208251612e-05, + "max_abs": 0.0019119152639177628, + "worst_column_rms": 0.0006550130820460575, + "worst_column": 1968, + "correctly_rounded": 131058, + "outputs": 131072 + }, + "mlx": { + "rms": 0.00010076847313642905, + "max_abs": 0.001994334736082237, + "worst_column_rms": 0.0006696350049925088, + "worst_column": 1968, + "correctly_rounded": 89989, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "7dd67e902a2eef702d22c1f1ea921c09e4ee7d054d66e5c7d9e2cbb7ecaa31fb" + }, + { + "mode": "decode", + "native": { + "rms": 6.414859585365679e-05, + "max_abs": 0.00048112778222275665, + "worst_column_rms": 0.00048112778222275665, + "worst_column": 670, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 6.414859585365679e-05, + "max_abs": 0.00048112778222275665, + "worst_column_rms": 0.00048112778222275665, + "worst_column": 670, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "mlx": { + "rms": 6.414859585365679e-05, + "max_abs": 0.00048112778222275665, + "worst_column_rms": 0.00048112778222275665, + "worst_column": 670, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "731b98f325784057f48d09e9eb0e5b87f1c2e4e1089729873689fea506272ca7" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-6bit", + "module": "language_model.model.layers.2.self_attn.q_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "VL-2B-6-attention_q.bf16", + "one_row_file": "VL-2B-6-attention_q-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 2048 + }, + { + "name": "VL-2B-6-attention_k", + "class": "attention_k", + "shape": [ + 64, + 2048, + 512 + ], + "input_sha256": "35ec818b558d14a1b2243b8760ed0fd8bda97895272ea6ced3cc5ae6dcb5f0c9", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 8.906782257355343e-05, + "max_abs": 0.0009527245338176726, + "worst_column_rms": 0.0003749029434800018, + "worst_column": 158, + "correctly_rounded": 32768, + "outputs": 32768 + }, + "original_native": { + "rms": 8.906782257357769e-05, + "max_abs": 0.0009527245338176726, + "worst_column_rms": 0.0003749029434800018, + "worst_column": 158, + "correctly_rounded": 32766, + "outputs": 32768 + }, + "mlx": { + "rms": 9.809270937773964e-05, + "max_abs": 0.0010065304195450153, + "worst_column_rms": 0.0003870446353198562, + "worst_column": 158, + "correctly_rounded": 23141, + "outputs": 32768 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "ec9d623b94b96e7f16efd8b78fbeb6594daef7712d34101e8171efb9d9a07e76" + }, + { + "mode": "decode", + "native": { + "rms": 7.278096825542681e-05, + "max_abs": 0.0004867799343628576, + "worst_column_rms": 0.0004867799343628576, + "worst_column": 158, + "correctly_rounded": 512, + "outputs": 512 + }, + "original_native": { + "rms": 7.278096825542681e-05, + "max_abs": 0.0004867799343628576, + "worst_column_rms": 0.0004867799343628576, + "worst_column": 158, + "correctly_rounded": 512, + "outputs": 512 + }, + "mlx": { + "rms": 7.278096825542681e-05, + "max_abs": 0.0004867799343628576, + "worst_column_rms": 0.0004867799343628576, + "worst_column": 158, + "correctly_rounded": 512, + "outputs": 512 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "897083fbf229dca38a8de1d30fac9a6098d55ec30874d2ba14a692d34bddb436" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-6bit", + "module": "language_model.model.layers.2.self_attn.k_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "VL-2B-6-attention_k.bf16", + "one_row_file": "VL-2B-6-attention_k-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 512 + }, + { + "name": "VL-2B-6-attention_v", + "class": "attention_v", + "shape": [ + 64, + 2048, + 512 + ], + "input_sha256": "35ec818b558d14a1b2243b8760ed0fd8bda97895272ea6ced3cc5ae6dcb5f0c9", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 7.31186477056622e-05, + "max_abs": 0.0012254196517460514, + "worst_column_rms": 0.00032863515977423516, + "worst_column": 87, + "correctly_rounded": 32768, + "outputs": 32768 + }, + "original_native": { + "rms": 7.311864773619243e-05, + "max_abs": 0.0012254196517460514, + "worst_column_rms": 0.00032863515977423516, + "worst_column": 87, + "correctly_rounded": 32763, + "outputs": 32768 + }, + "mlx": { + "rms": 8.634597479927417e-05, + "max_abs": 0.0012254196517460514, + "worst_column_rms": 0.0003324614844358142, + "worst_column": 87, + "correctly_rounded": 21360, + "outputs": 32768 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "c9767157863db3b698250112aedfbf14601174a6e1bde137f72881df5f50c1a3" + }, + { + "mode": "decode", + "native": { + "rms": 7.948639484692553e-05, + "max_abs": 0.0004654345611925237, + "worst_column_rms": 0.0004654345611925237, + "worst_column": 233, + "correctly_rounded": 512, + "outputs": 512 + }, + "original_native": { + "rms": 7.948639484692553e-05, + "max_abs": 0.0004654345611925237, + "worst_column_rms": 0.0004654345611925237, + "worst_column": 233, + "correctly_rounded": 512, + "outputs": 512 + }, + "mlx": { + "rms": 7.948639484692553e-05, + "max_abs": 0.0004654345611925237, + "worst_column_rms": 0.0004654345611925237, + "worst_column": 233, + "correctly_rounded": 512, + "outputs": 512 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "a8d94974c7616852ac893dd29ba55aa7bf8e82c9760dd7793157eb96df59358f" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-6bit", + "module": "language_model.model.layers.2.self_attn.v_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "VL-2B-6-attention_v.bf16", + "one_row_file": "VL-2B-6-attention_v-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 512 + }, + { + "name": "VL-2B-6-attention_gate", + "class": "attention_gate", + "shape": [ + 64, + 2048, + 16 + ], + "input_sha256": "35ec818b558d14a1b2243b8760ed0fd8bda97895272ea6ced3cc5ae6dcb5f0c9", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.0012872964762039862, + "max_abs": 0.00389142816857202, + "worst_column_rms": 0.0023626233684481517, + "worst_column": 7, + "correctly_rounded": 1024, + "outputs": 1024 + }, + "original_native": { + "rms": 0.0012872964762039862, + "max_abs": 0.00389142816857202, + "worst_column_rms": 0.0023626233684481517, + "worst_column": 7, + "correctly_rounded": 1024, + "outputs": 1024 + }, + "mlx": { + "rms": 0.0013106506150646667, + "max_abs": 0.004121785292227287, + "worst_column_rms": 0.0023928920553825434, + "worst_column": 7, + "correctly_rounded": 938, + "outputs": 1024 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "ae7f4042b17c5c7b77562de99ec66e581e5815e9f2e61d8cec0c461d7d0eb2cb" + }, + { + "mode": "decode", + "native": { + "rms": 0.0006111186468776001, + "max_abs": 0.0011886373395100236, + "worst_column_rms": 0.0011886373395100236, + "worst_column": 11, + "correctly_rounded": 16, + "outputs": 16 + }, + "original_native": { + "rms": 0.0006111186468776001, + "max_abs": 0.0011886373395100236, + "worst_column_rms": 0.0011886373395100236, + "worst_column": 11, + "correctly_rounded": 16, + "outputs": 16 + }, + "mlx": { + "rms": 0.0006111186468776001, + "max_abs": 0.0011886373395100236, + "worst_column_rms": 0.0011886373395100236, + "worst_column": 11, + "correctly_rounded": 16, + "outputs": 16 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "0e7e3e65904ebd568f7b50e3856311cd08f4ce5bd2b5c25e7cccaeb04bf1d2ad" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-6bit", + "module": "language_model.model.layers.2.self_attn.gate_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "VL-2B-6-attention_gate.bf16", + "one_row_file": "VL-2B-6-attention_gate-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 16 + }, + { + "name": "VL-2B-6-attention_out", + "class": "attention_out", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "72d4db02145b71840cc3d8e2a4d4d566e6535353bd7eb31fcb4a8adaaf7c826a", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 1.5958829736153112e-05, + "max_abs": 0.00047174606697808485, + "worst_column_rms": 0.00016858932862929868, + "worst_column": 1271, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 1.595882980948954e-05, + "max_abs": 0.00047174606697808485, + "worst_column_rms": 0.00016858932862929868, + "worst_column": 1271, + "correctly_rounded": 131053, + "outputs": 131072 + }, + "mlx": { + "rms": 2.034130507362192e-05, + "max_abs": 0.0005048164330219151, + "worst_column_rms": 0.00016957458851323386, + "worst_column": 1271, + "correctly_rounded": 76175, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "df60fd6d1502456327eda9c18e9a1c8f083f57f168347c50ed13cbf327386c92" + }, + { + "mode": "decode", + "native": { + "rms": 2.8871335042799292e-05, + "max_abs": 0.00035475015465635806, + "worst_column_rms": 0.00035475015465635806, + "worst_column": 2039, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 2.8871335042799292e-05, + "max_abs": 0.00035475015465635806, + "worst_column_rms": 0.00035475015465635806, + "worst_column": 2039, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "mlx": { + "rms": 2.8871335042818774e-05, + "max_abs": 0.00035475015465635806, + "worst_column_rms": 0.00035475015465635806, + "worst_column": 2039, + "correctly_rounded": 2047, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "12f97abed04460ce87b0fedebcae80babd945ccb9cde3de78e704f17cb224e9e" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-6bit", + "module": "language_model.model.layers.2.self_attn.o_proj", + "input_stage": "layer_02_attn_gated", + "input_file": "VL-2B-6-attention_out.bf16", + "one_row_file": "VL-2B-6-attention_out-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 2048 + }, + { + "name": "VL-2B-6-vocabulary_head", + "class": "vocabulary_head", + "shape": [ + 64, + 2048, + 128000 + ], + "input_sha256": "29275ae68a6a32e47063f646d0b042f4acbe02e8b85bbfa856169b4e7b0965a8", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.006373968419129721, + "max_abs": 0.06070356583222747, + "worst_column_rms": 0.01722533584007339, + "worst_column": 20, + "correctly_rounded": 8192000, + "outputs": 8192000 + }, + "original_native": { + "rms": 0.006373968427341432, + "max_abs": 0.06070356583222747, + "worst_column_rms": 0.01722533584007339, + "worst_column": 20, + "correctly_rounded": 8191595, + "outputs": 8192000 + }, + "mlx": { + "rms": 0.007326178382650345, + "max_abs": 0.06429643416777253, + "worst_column_rms": 0.017961569007072145, + "worst_column": 20, + "correctly_rounded": 6323634, + "outputs": 8192000 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "a2cdac531945e51aa61719171460f3e40cf8b658f6f4532dd46fa944b10483ad" + }, + { + "mode": "decode", + "native": { + "rms": 0.005466403479383152, + "max_abs": 0.023867991287261248, + "worst_column_rms": 0.023867991287261248, + "worst_column": 1492, + "correctly_rounded": 128000, + "outputs": 128000 + }, + "original_native": { + "rms": 0.005466403479812204, + "max_abs": 0.023867991287261248, + "worst_column_rms": 0.023867991287261248, + "worst_column": 1492, + "correctly_rounded": 127998, + "outputs": 128000 + }, + "mlx": { + "rms": 0.005466403480440142, + "max_abs": 0.023867991287261248, + "worst_column_rms": 0.023867991287261248, + "worst_column": 1492, + "correctly_rounded": 127993, + "outputs": 128000 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "3af09e6edb5b63ee9acb212a1fe4f23d82d7dfdf39ae63617c6fb60ebc523362" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-6bit", + "module": "language_model.model.embed_tokens", + "input_stage": "final_norm", + "input_file": "VL-2B-6-vocabulary_head.bf16", + "one_row_file": "VL-2B-6-vocabulary_head-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-6bit", + "rows": 64, + "k": 2048, + "n": 128000 + }, + { + "name": "VL-2B-8-conv_in", + "class": "conv_in", + "shape": [ + 64, + 2048, + 6144 + ], + "input_sha256": "de85672976434ce7521a576d07b5c737c0d7717598181181666d4a52e66d3a88", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.00030148620224470965, + "max_abs": 0.005836656955580111, + "worst_column_rms": 0.0016355073068322823, + "worst_column": 3899, + "correctly_rounded": 393216, + "outputs": 393216 + }, + "original_native": { + "rms": 0.00030148620236452505, + "max_abs": 0.005836656955580111, + "worst_column_rms": 0.0016355073068322823, + "worst_column": 3899, + "correctly_rounded": 393173, + "outputs": 393216 + }, + "mlx": { + "rms": 0.0003888308254313817, + "max_abs": 0.005836656955580111, + "worst_column_rms": 0.0016639633981208764, + "worst_column": 3899, + "correctly_rounded": 222514, + "outputs": 393216 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "71bf9b014f1cc1b8edc7167067e790c3a5e536a88109751b5ed4f09528098597" + }, + { + "mode": "decode", + "native": { + "rms": 0.0003453614152093189, + "max_abs": 0.001950271922396496, + "worst_column_rms": 0.001950271922396496, + "worst_column": 4791, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "original_native": { + "rms": 0.0003453614152093189, + "max_abs": 0.001950271922396496, + "worst_column_rms": 0.001950271922396496, + "worst_column": 4791, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "mlx": { + "rms": 0.0003453614152093189, + "max_abs": 0.001950271922396496, + "worst_column_rms": 0.001950271922396496, + "worst_column": 4791, + "correctly_rounded": 6144, + "outputs": 6144 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "abe2d50106ca7025d34167f9b593f034e3cccfd56c2aa5473efa2505b6469c9a" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-8bit", + "module": "language_model.model.layers.0.conv.in_proj", + "input_stage": "layer_00_operator_norm", + "input_file": "VL-2B-8-conv_in.bf16", + "one_row_file": "VL-2B-8-conv_in-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 6144 + }, + { + "name": "VL-2B-8-conv_out", + "class": "conv_out", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "5e8b4fba6eefd14cfe7b11ed848fea55c19612db9a25607da2b63e858f27f328", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 6.779457097799859e-06, + "max_abs": 0.0004822275076428184, + "worst_column_rms": 0.00015431750922573976, + "worst_column": 1688, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 6.779457099182695e-06, + "max_abs": 0.0004822275076428184, + "worst_column_rms": 0.00015431750922573976, + "worst_column": 1688, + "correctly_rounded": 131042, + "outputs": 131072 + }, + "mlx": { + "rms": 7.097410383171021e-06, + "max_abs": 0.0004822275076428184, + "worst_column_rms": 0.00015431750922573976, + "worst_column": 1688, + "correctly_rounded": 84905, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "3a88ff8d8fb1b7d99092f109dedf5f81ca6d76c32433c9b6aeb3596b7f6ab8b2" + }, + { + "mode": "decode", + "native": { + "rms": 1.870317366158558e-06, + "max_abs": 4.7558576221396615e-05, + "worst_column_rms": 4.7558576221396615e-05, + "worst_column": 42, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 1.8703173945178206e-06, + "max_abs": 4.7558576221396615e-05, + "worst_column_rms": 4.7558576221396615e-05, + "worst_column": 42, + "correctly_rounded": 2047, + "outputs": 2048 + }, + "mlx": { + "rms": 1.8703173664541996e-06, + "max_abs": 4.7558576221396615e-05, + "worst_column_rms": 4.7558576221396615e-05, + "worst_column": 42, + "correctly_rounded": 2046, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "d32cc9511e429d97a0c492a42d9b7b6184ad6165ac1ed9e3b355ed6765af85fc" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-8bit", + "module": "language_model.model.layers.0.conv.out_proj", + "input_stage": "layer_00_conv_gated", + "input_file": "VL-2B-8-conv_out.bf16", + "one_row_file": "VL-2B-8-conv_out-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 2048 + }, + { + "name": "VL-2B-8-ffn_gate", + "class": "ffn_gate", + "shape": [ + 64, + 2048, + 7680 + ], + "input_sha256": "7d8cf3f37dc15f6ab8c12bde40c8f6c7586cfb785c402dfa2f363ffbb2b5eceb", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.000294705459081744, + "max_abs": 0.003880211493196839, + "worst_column_rms": 0.002590423432140416, + "worst_column": 4796, + "correctly_rounded": 491520, + "outputs": 491520 + }, + "original_native": { + "rms": 0.0002947054592933254, + "max_abs": 0.003880211493196839, + "worst_column_rms": 0.002590423432140416, + "worst_column": 4796, + "correctly_rounded": 491479, + "outputs": 491520 + }, + "mlx": { + "rms": 0.00034375196860269317, + "max_abs": 0.004105369709577644, + "worst_column_rms": 0.002590423432140416, + "worst_column": 4796, + "correctly_rounded": 343797, + "outputs": 491520 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "e0b4f14b9e9b91516bd391a06838292f5ff79ae9cbc429fe404bde9f5a130e08" + }, + { + "mode": "decode", + "native": { + "rms": 0.0001728576095401693, + "max_abs": 0.0014598445486626588, + "worst_column_rms": 0.0014598445486626588, + "worst_column": 1709, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "original_native": { + "rms": 0.0001728576095419242, + "max_abs": 0.0014598445486626588, + "worst_column_rms": 0.0014598445486626588, + "worst_column": 1709, + "correctly_rounded": 7679, + "outputs": 7680 + }, + "mlx": { + "rms": 0.0001728576095419242, + "max_abs": 0.0014598445486626588, + "worst_column_rms": 0.0014598445486626588, + "worst_column": 1709, + "correctly_rounded": 7679, + "outputs": 7680 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "f5caab73d84c4262377769654faaba8c285b694455e329264919a3f9641db7ef" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-8bit", + "module": "language_model.model.layers.0.feed_forward.w1", + "input_stage": "layer_00_ffn_norm", + "input_file": "VL-2B-8-ffn_gate.bf16", + "one_row_file": "VL-2B-8-ffn_gate-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 7680 + }, + { + "name": "VL-2B-8-ffn_up", + "class": "ffn_up", + "shape": [ + 64, + 2048, + 7680 + ], + "input_sha256": "7d8cf3f37dc15f6ab8c12bde40c8f6c7586cfb785c402dfa2f363ffbb2b5eceb", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.00017504776791043886, + "max_abs": 0.0019298178849567194, + "worst_column_rms": 0.0007048199055249673, + "worst_column": 6325, + "correctly_rounded": 491520, + "outputs": 491520 + }, + "original_native": { + "rms": 0.00017504776833737339, + "max_abs": 0.0019298178849567194, + "worst_column_rms": 0.0007048199055249673, + "worst_column": 6325, + "correctly_rounded": 491471, + "outputs": 491520 + }, + "mlx": { + "rms": 0.0002370633985352454, + "max_abs": 0.0020002248129458167, + "worst_column_rms": 0.0007048199055249673, + "worst_column": 6325, + "correctly_rounded": 287368, + "outputs": 491520 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "fcce4ac31aa0f9ef38b3cd0cb7c25398dcf6ed68a309362418b5814af62eca3e" + }, + { + "mode": "decode", + "native": { + "rms": 0.00015871779938016168, + "max_abs": 0.0009648259365349077, + "worst_column_rms": 0.0009648259365349077, + "worst_column": 2925, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "original_native": { + "rms": 0.00015871779938016168, + "max_abs": 0.0009648259365349077, + "worst_column_rms": 0.0009648259365349077, + "worst_column": 2925, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "mlx": { + "rms": 0.00015871779938016168, + "max_abs": 0.0009648259365349077, + "worst_column_rms": 0.0009648259365349077, + "worst_column": 2925, + "correctly_rounded": 7680, + "outputs": 7680 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "112d5b19e6a549b3ea21198dc4f933dbaeeb6f3579cc895c076c6b7f406cd105" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-8bit", + "module": "language_model.model.layers.0.feed_forward.w3", + "input_stage": "layer_00_ffn_norm", + "input_file": "VL-2B-8-ffn_up.bf16", + "one_row_file": "VL-2B-8-ffn_up-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 7680 + }, + { + "name": "VL-2B-8-ffn_down", + "class": "ffn_down", + "shape": [ + 64, + 7680, + 2048 + ], + "input_sha256": "72f501f4d99cb3c3efe9c153b2e038492c43e765711113bad9ccc4e13712ef5a", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 8.563944258886382e-05, + "max_abs": 0.0038581252033260682, + "worst_column_rms": 0.0016558300621447907, + "worst_column": 108, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 8.563944259500281e-05, + "max_abs": 0.0038581252033260682, + "worst_column_rms": 0.0016558300621447907, + "worst_column": 108, + "correctly_rounded": 131064, + "outputs": 131072 + }, + "mlx": { + "rms": 8.891713884334553e-05, + "max_abs": 0.003954374796673932, + "worst_column_rms": 0.0016560075735526972, + "worst_column": 108, + "correctly_rounded": 86728, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "6896f97343dfd72dfcdb74d4f1f5141bd4283249844922b8def07210c5abf213" + }, + { + "mode": "decode", + "native": { + "rms": 2.532296764277322e-05, + "max_abs": 0.0003554259213336408, + "worst_column_rms": 0.0003554259213336408, + "worst_column": 169, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 2.532296764277322e-05, + "max_abs": 0.0003554259213336408, + "worst_column_rms": 0.0003554259213336408, + "worst_column": 169, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "mlx": { + "rms": 2.5322967643350605e-05, + "max_abs": 0.0003554259213336408, + "worst_column_rms": 0.0003554259213336408, + "worst_column": 169, + "correctly_rounded": 2047, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "6fa80d2c836c95a713b9a496e3d75954b69f48c775fe975f6011f3a9583f0870" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-8bit", + "module": "language_model.model.layers.0.feed_forward.w2", + "input_stage": "layer_00_act", + "input_file": "VL-2B-8-ffn_down.bf16", + "one_row_file": "VL-2B-8-ffn_down-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-8bit", + "rows": 64, + "k": 7680, + "n": 2048 + }, + { + "name": "VL-2B-8-attention_q", + "class": "attention_q", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "4e0b08dc7cca474bcfb6d052a9cdd977b7ea8ae3eb08410eb1b6d453f3c34d30", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 9.057493846422864e-05, + "max_abs": 0.0018691720088099828, + "worst_column_rms": 0.0007381820480928835, + "worst_column": 1968, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 9.057493852295167e-05, + "max_abs": 0.0018691720088099828, + "worst_column_rms": 0.0007381820480928835, + "worst_column": 1968, + "correctly_rounded": 131052, + "outputs": 131072 + }, + "mlx": { + "rms": 0.00010127549963509076, + "max_abs": 0.002037077991190017, + "worst_column_rms": 0.000761227824594019, + "worst_column": 1968, + "correctly_rounded": 89200, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "2406496d7aac10e40363859bcbe52cdcbbd424d233cedd915f90ea89bbc351f4" + }, + { + "mode": "decode", + "native": { + "rms": 6.666698360690884e-05, + "max_abs": 0.0009607105021132156, + "worst_column_rms": 0.0009607105021132156, + "worst_column": 1968, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 6.666698361584077e-05, + "max_abs": 0.0009607105021132156, + "worst_column_rms": 0.0009607105021132156, + "worst_column": 1968, + "correctly_rounded": 2047, + "outputs": 2048 + }, + "mlx": { + "rms": 6.666698360690884e-05, + "max_abs": 0.0009607105021132156, + "worst_column_rms": 0.0009607105021132156, + "worst_column": 1968, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "7ebc981eadeeb63851ef1ac62366369946d4cd85699291ef54ea48ae45b9ed9c" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-8bit", + "module": "language_model.model.layers.2.self_attn.q_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "VL-2B-8-attention_q.bf16", + "one_row_file": "VL-2B-8-attention_q-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 2048 + }, + { + "name": "VL-2B-8-attention_k", + "class": "attention_k", + "shape": [ + 64, + 2048, + 512 + ], + "input_sha256": "4e0b08dc7cca474bcfb6d052a9cdd977b7ea8ae3eb08410eb1b6d453f3c34d30", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 9.106405386602432e-05, + "max_abs": 0.0009612401697722817, + "worst_column_rms": 0.00042010348610056053, + "worst_column": 223, + "correctly_rounded": 32768, + "outputs": 32768 + }, + "original_native": { + "rms": 9.106405386602432e-05, + "max_abs": 0.0009612401697722817, + "worst_column_rms": 0.00042010348610056053, + "worst_column": 223, + "correctly_rounded": 32768, + "outputs": 32768 + }, + "mlx": { + "rms": 0.00010135598659951092, + "max_abs": 0.0009612401697722817, + "worst_column_rms": 0.000422906794157311, + "worst_column": 223, + "correctly_rounded": 22900, + "outputs": 32768 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "e04f2457b18a1378260dcc1ab3f6372c8f82a54c65b95ca3b95575fc36798874" + }, + { + "mode": "decode", + "native": { + "rms": 7.523138396442872e-05, + "max_abs": 0.000461977266240865, + "worst_column_rms": 0.000461977266240865, + "worst_column": 158, + "correctly_rounded": 512, + "outputs": 512 + }, + "original_native": { + "rms": 7.523138396442872e-05, + "max_abs": 0.000461977266240865, + "worst_column_rms": 0.000461977266240865, + "worst_column": 158, + "correctly_rounded": 512, + "outputs": 512 + }, + "mlx": { + "rms": 7.523138396442872e-05, + "max_abs": 0.000461977266240865, + "worst_column_rms": 0.000461977266240865, + "worst_column": 158, + "correctly_rounded": 512, + "outputs": 512 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "ca775c01620b1478372d4823c64a976b5e0ef3ad0d3f1c4d90e731cc4f8671a9" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-8bit", + "module": "language_model.model.layers.2.self_attn.k_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "VL-2B-8-attention_k.bf16", + "one_row_file": "VL-2B-8-attention_k-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 512 + }, + { + "name": "VL-2B-8-attention_v", + "class": "attention_v", + "shape": [ + 64, + 2048, + 512 + ], + "input_sha256": "4e0b08dc7cca474bcfb6d052a9cdd977b7ea8ae3eb08410eb1b6d453f3c34d30", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 7.422960674570326e-05, + "max_abs": 0.0009482959367232979, + "worst_column_rms": 0.0002873907319064168, + "worst_column": 87, + "correctly_rounded": 32768, + "outputs": 32768 + }, + "original_native": { + "rms": 7.42296067574851e-05, + "max_abs": 0.0009482959367232979, + "worst_column_rms": 0.0002873907319064168, + "worst_column": 87, + "correctly_rounded": 32764, + "outputs": 32768 + }, + "mlx": { + "rms": 8.859472446413804e-05, + "max_abs": 0.0009482959367232979, + "worst_column_rms": 0.00028902520423560227, + "worst_column": 87, + "correctly_rounded": 20898, + "outputs": 32768 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "3624b048e8946768c2f095071dd36e53475847d188b87da9146ae6d9fe064040" + }, + { + "mode": "decode", + "native": { + "rms": 7.658372984501761e-05, + "max_abs": 0.00041831859562080353, + "worst_column_rms": 0.00041831859562080353, + "worst_column": 65, + "correctly_rounded": 512, + "outputs": 512 + }, + "original_native": { + "rms": 7.658372984501761e-05, + "max_abs": 0.00041831859562080353, + "worst_column_rms": 0.00041831859562080353, + "worst_column": 65, + "correctly_rounded": 512, + "outputs": 512 + }, + "mlx": { + "rms": 7.658372984501761e-05, + "max_abs": 0.00041831859562080353, + "worst_column_rms": 0.00041831859562080353, + "worst_column": 65, + "correctly_rounded": 512, + "outputs": 512 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "861913b23c3fcbafe53476e6537fe0c03da18bb338a6d50842dd13f7e84376e7" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-8bit", + "module": "language_model.model.layers.2.self_attn.v_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "VL-2B-8-attention_v.bf16", + "one_row_file": "VL-2B-8-attention_v-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 512 + }, + { + "name": "VL-2B-8-attention_gate", + "class": "attention_gate", + "shape": [ + 64, + 2048, + 16 + ], + "input_sha256": "4e0b08dc7cca474bcfb6d052a9cdd977b7ea8ae3eb08410eb1b6d453f3c34d30", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.0013454854557382629, + "max_abs": 0.0038619576407654677, + "worst_column_rms": 0.0021855458232628473, + "worst_column": 0, + "correctly_rounded": 1024, + "outputs": 1024 + }, + "original_native": { + "rms": 0.0013454854557382629, + "max_abs": 0.0038619576407654677, + "worst_column_rms": 0.0021855458232628473, + "worst_column": 0, + "correctly_rounded": 1024, + "outputs": 1024 + }, + "mlx": { + "rms": 0.0013607512162365338, + "max_abs": 0.0040769908191578, + "worst_column_rms": 0.0022154594050187537, + "worst_column": 0, + "correctly_rounded": 945, + "outputs": 1024 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "5ac650fc043aaa905783be602dfbddbc6a8f0c0a94d78d7188743bc70801508e" + }, + { + "mode": "decode", + "native": { + "rms": 0.0005791900157388877, + "max_abs": 0.0016601522584096529, + "worst_column_rms": 0.0016601522584096529, + "worst_column": 11, + "correctly_rounded": 16, + "outputs": 16 + }, + "original_native": { + "rms": 0.0005791900157388877, + "max_abs": 0.0016601522584096529, + "worst_column_rms": 0.0016601522584096529, + "worst_column": 11, + "correctly_rounded": 16, + "outputs": 16 + }, + "mlx": { + "rms": 0.0005791900157388877, + "max_abs": 0.0016601522584096529, + "worst_column_rms": 0.0016601522584096529, + "worst_column": 11, + "correctly_rounded": 16, + "outputs": 16 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "95cbe3c3772e06917f9b2aa90cafd9ffd5cbeacf97a131ed3a77862627308818" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-8bit", + "module": "language_model.model.layers.2.self_attn.gate_proj", + "input_stage": "layer_02_operator_norm", + "input_file": "VL-2B-8-attention_gate.bf16", + "one_row_file": "VL-2B-8-attention_gate-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 16 + }, + { + "name": "VL-2B-8-attention_out", + "class": "attention_out", + "shape": [ + 64, + 2048, + 2048 + ], + "input_sha256": "7e242a975120773f32c7094c5ed172bdc58834bbae12e2c0d68b98c89c0cffe4", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 1.6033544964160292e-05, + "max_abs": 0.0004833645248094598, + "worst_column_rms": 0.00018773029281282406, + "worst_column": 1918, + "correctly_rounded": 131072, + "outputs": 131072 + }, + "original_native": { + "rms": 1.6033544979170126e-05, + "max_abs": 0.0004833645248094598, + "worst_column_rms": 0.00018773029281282406, + "worst_column": 1918, + "correctly_rounded": 131060, + "outputs": 131072 + }, + "mlx": { + "rms": 2.0689582560999675e-05, + "max_abs": 0.0005244625985625362, + "worst_column_rms": 0.0001897514714654325, + "worst_column": 1271, + "correctly_rounded": 75390, + "outputs": 131072 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "0c026f593adfcb6b3ed08c37bebbfc5d0cfc265eb1e6bf4f2b3dd3af8d3de347" + }, + { + "mode": "decode", + "native": { + "rms": 2.7021030129146707e-05, + "max_abs": 0.000298451392154675, + "worst_column_rms": 0.000298451392154675, + "worst_column": 1996, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "original_native": { + "rms": 2.7021030129146707e-05, + "max_abs": 0.000298451392154675, + "worst_column_rms": 0.000298451392154675, + "worst_column": 1996, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "mlx": { + "rms": 2.7021030129146707e-05, + "max_abs": 0.000298451392154675, + "worst_column_rms": 0.000298451392154675, + "worst_column": 1996, + "correctly_rounded": 2048, + "outputs": 2048 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "e5632e43d6d113a76edddfecc4f162eacc1c6c407b8dcf58b35487528da6e1e1" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-8bit", + "module": "language_model.model.layers.2.self_attn.o_proj", + "input_stage": "layer_02_attn_gated", + "input_file": "VL-2B-8-attention_out.bf16", + "one_row_file": "VL-2B-8-attention_out-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 2048 + }, + { + "name": "VL-2B-8-vocabulary_head", + "class": "vocabulary_head", + "shape": [ + 64, + 2048, + 128000 + ], + "input_sha256": "69e9450f2984d88628637b19bfd27885d6e09ebe0841a6c99a60340fc2a6c237", + "comparisons": [ + { + "mode": "prompt", + "native": { + "rms": 0.00637062298687024, + "max_abs": 0.06121470243670046, + "worst_column_rms": 0.01631051233544983, + "worst_column": 278, + "correctly_rounded": 8192000, + "outputs": 8192000 + }, + "original_native": { + "rms": 0.006370622995905622, + "max_abs": 0.06121470243670046, + "worst_column_rms": 0.01631051233544983, + "worst_column": 278, + "correctly_rounded": 8191553, + "outputs": 8192000 + }, + "mlx": { + "rms": 0.007425419414880146, + "max_abs": 0.06121470243670046, + "worst_column_rms": 0.016382519090016284, + "worst_column": 278, + "correctly_rounded": 6260448, + "outputs": 8192000 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "61f3c553b0a0daf33f2dd29f46677112dae2639fd338014b7ebdd22558c6b1cb" + }, + { + "mode": "decode", + "native": { + "rms": 0.00450617515076936, + "max_abs": 0.02677314502216177, + "worst_column_rms": 0.02677314502216177, + "worst_column": 1022, + "correctly_rounded": 128000, + "outputs": 128000 + }, + "original_native": { + "rms": 0.0045061751651349746, + "max_abs": 0.02677314502216177, + "worst_column_rms": 0.02677314502216177, + "worst_column": 1022, + "correctly_rounded": 127992, + "outputs": 128000 + }, + "mlx": { + "rms": 0.004506175151630346, + "max_abs": 0.02677314502216177, + "worst_column_rms": 0.02677314502216177, + "worst_column": 1022, + "correctly_rounded": 127994, + "outputs": 128000 + }, + "native_columns_worse_than_original": 0, + "native_columns_worse_than_mlx": 0, + "output_sha256": "e250665cdfd5e271e66e762ae85ec1bae526ef802cc62ec9f736be9d52622f13" + } + ], + "prompt_decode_equal": true, + "checkpoint": "LFM3-VL-2B-MLX-8bit", + "module": "language_model.model.embed_tokens", + "input_stage": "final_norm", + "input_file": "VL-2B-8-vocabulary_head.bf16", + "one_row_file": "VL-2B-8-vocabulary_head-one.bf16", + "model_directory": "LFM3-VL-2B-MLX-8bit", + "rows": 64, + "k": 2048, + "n": 128000 + } + ] +} diff --git a/docs/receipts/lfm3/binned-row-twin.json b/docs/receipts/lfm3/binned-row-twin.json new file mode 100644 index 0000000000..a1e8140f73 --- /dev/null +++ b/docs/receipts/lfm3/binned-row-twin.json @@ -0,0 +1,941 @@ +{ + "scope": "Explicitly opt-in canonical row twin; ordinary model loading remains disabled. No served performance pass.", + "commit": "5a2bdea76", + "comparisons": 132, + "bytes": 123814080, + "cases": [ + { + "name": "1.7B-4-conv_in", + "rows": 64, + "bytes": 786432, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "764c618b648c35902ec16e7f8578687e9b514a4c06ab5e7191c122ce8c704ffe" + }, + { + "name": "1.7B-4-conv_in", + "rows": 1, + "bytes": 12288, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "fa0a8412931265b2da7c5083d58a2bb9f9a776a65f8476ee623a7293dc07a1d2" + }, + { + "name": "1.7B-4-conv_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "72ccfcb68f3a6e49a45a3c61501b886efdf56617095f4ea39c6f8fa6dacf812c" + }, + { + "name": "1.7B-4-conv_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "571e0e60f83da426e1f36973ce381c6c07dc0250805f58875cb4c6ce3b16f635" + }, + { + "name": "1.7B-4-ffn_gate", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "c2f30477ae8dc9605d0624cd0e488921a3d628f5e22756d0d699ceea360166a3" + }, + { + "name": "1.7B-4-ffn_gate", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "88fe235008e7a04f2aa0a365a599700df32f4e4b511487ae47f84cbd9042147d" + }, + { + "name": "1.7B-4-ffn_up", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "bfa00c6ce34772edc11f01c7de19f0b3c9c3b2cf28288500c70cc1a8fe823787" + }, + { + "name": "1.7B-4-ffn_up", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "7936b781b393ee17409bd70f5bc5592c64963a8d08141633b37e072a3c13362d" + }, + { + "name": "1.7B-4-ffn_down", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "81e564394395e6f5ae3e1ba6f1c524ea76ec070846e90020ae4db3e9a1d8f122" + }, + { + "name": "1.7B-4-ffn_down", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "c33b3ac7d6ecc10fc84d4c45adb28d52fd65037a2f34a2293123484fe9747e76" + }, + { + "name": "1.7B-4-attention_q", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "35d010f3010570b13756e02ffad15c2e3d88a20c8fa17d848fc9644e7e46f41f" + }, + { + "name": "1.7B-4-attention_q", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "a2a0743f91f9e9b823701b82fd82860e202ad3a82b6c94cd8b8a200f5a48f86c" + }, + { + "name": "1.7B-4-attention_k", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "b6119e6fdfa2e6b2e57e63a5efbdde3777ea84cebadd42f70b8eb2d24a0b38c6" + }, + { + "name": "1.7B-4-attention_k", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "49cd1a700b8e55f3ae9081f9dfec38426a2c42cfd091122200705bc882dde26e" + }, + { + "name": "1.7B-4-attention_v", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "c8e358a2139f014b68ef723b2ad0b600422c1600c93dd497e42818487a4cf8b1" + }, + { + "name": "1.7B-4-attention_v", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "f807cedaf1556dc08feeb85ab4e26c8e9adb2597a9a27f7cce06259b40e95643" + }, + { + "name": "1.7B-4-attention_gate", + "rows": 64, + "bytes": 2048, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "acaf96d0437193e28abe706219a35cd47abb6783f846a6c3edcf376c73ebbbd5" + }, + { + "name": "1.7B-4-attention_gate", + "rows": 1, + "bytes": 32, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "04c0a5e4019680d000290cddf3e2a236028ab4f341059eab523bab6f452402d3" + }, + { + "name": "1.7B-4-attention_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "fcc3661b25c7a9b58e7cf37790af16a9f9775061dc504e8880574cf5b3064509" + }, + { + "name": "1.7B-4-attention_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "9bc91b6132035b4c9c3e26eb2b0e1ff0471e8b1675f0de175c4555e4e3e4a6d3" + }, + { + "name": "1.7B-4-vocabulary_head", + "rows": 64, + "bytes": 16384000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "6786268290e834bbc1d6f7367f17f3d8240fede1a1c66608edf843bdd3ddd3b9" + }, + { + "name": "1.7B-4-vocabulary_head", + "rows": 1, + "bytes": 256000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "ab9a7a2a9c4ce1da97742a4d0d5bfd497ee5f97be6e7f40ddbb35074779490f1" + }, + { + "name": "1.7B-6-conv_in", + "rows": 64, + "bytes": 786432, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "e690ade98b46e3c1ed023f33a6c729070b280b6b367b6356d046bad571f9e7d5" + }, + { + "name": "1.7B-6-conv_in", + "rows": 1, + "bytes": 12288, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "8812effabc5786e48e33c14f58aa0aed55ba1a2041ff8c68aa7c0e57c1d5db14" + }, + { + "name": "1.7B-6-conv_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "7b42d71bdf4c84a41f239b9928d62a42d591e1a19a32595add3b17bb0402ec08" + }, + { + "name": "1.7B-6-conv_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "6b6acdf7bfcca8b64f078b9bc8ce48a03490595f2870672482e1509380bcd107" + }, + { + "name": "1.7B-6-ffn_gate", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "6ff356e3c153ebc2efebc895be224be29f28512a97f76eaee2e29b20c56aab32" + }, + { + "name": "1.7B-6-ffn_gate", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "dc997a64e86ef2c76b962890db232dddd6bdd55819092a846abd0b0f5b82e6f3" + }, + { + "name": "1.7B-6-ffn_up", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "ad79c7c7bfd66a7545a9d76490566a10a3fd3dd0ee2fd74f97ca0e144a52f2f4" + }, + { + "name": "1.7B-6-ffn_up", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "546cd059b084c49f08ca8244d16b9ee8a5c2c4c797492adf45d359cb77ac141a" + }, + { + "name": "1.7B-6-ffn_down", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "3a6b7f54b9e6e33077d0bc94526b59f8bd460b4ef08470658b17df388abf6789" + }, + { + "name": "1.7B-6-ffn_down", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "d58d233e1dd4097bce3dda862fee878e15ffa51016c11d235f7544a7cc6e3db3" + }, + { + "name": "1.7B-6-attention_q", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "20779bc07a475a9ca2e2e43f9dfcc20e5e335854cb233f1633a45d8fa425140e" + }, + { + "name": "1.7B-6-attention_q", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "0bacb201fc20be38fbe4bc2589867d82c517ee19f3c261766f80d0f1f80c1cbf" + }, + { + "name": "1.7B-6-attention_k", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "92fef97597874698bad91388b36fcb1579fd9176643637397e43bca03a61d4e0" + }, + { + "name": "1.7B-6-attention_k", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "b59457203b990344119a76037c9ad8e4dbeaaa005239ef3d6c12d1dad72d229f" + }, + { + "name": "1.7B-6-attention_v", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "087f15e21a21ff48959028377610106d96c0bfd2f78e37ce6f039eaea7f95fb2" + }, + { + "name": "1.7B-6-attention_v", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "3b40c61d73782c5b2193028ea246f7964508017bf74fbf63e69750c99edce57a" + }, + { + "name": "1.7B-6-attention_gate", + "rows": 64, + "bytes": 2048, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "a71bd4049b62100ef10b401077aa7f1ee7b92cf39165bf06f217847b417f08c4" + }, + { + "name": "1.7B-6-attention_gate", + "rows": 1, + "bytes": 32, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "bd5cc9257542ff115afb330579f0fc9ca115c7b01bd446b1059f8e4ecbf2bba0" + }, + { + "name": "1.7B-6-attention_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "924ec41f566032e6928c145693aef559153b4cc4e0e3582002ca4cb3f1bb5574" + }, + { + "name": "1.7B-6-attention_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "53b6cdfc5c6ae2b1989b3250cb5fe665338d00065a9f340ee99b7878d3c796f9" + }, + { + "name": "1.7B-6-vocabulary_head", + "rows": 64, + "bytes": 16384000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "4aa176ed5fda5e148f8ae28a9aa7ab2476b91e447a1eb84d3c543a20f9fc043b" + }, + { + "name": "1.7B-6-vocabulary_head", + "rows": 1, + "bytes": 256000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "a7d12df1673fad09167e8ff16292935fc0eec2044e57119bdd14c9cf7ad914c1" + }, + { + "name": "1.7B-8-conv_in", + "rows": 64, + "bytes": 786432, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "d6b35c4877de2f0a8d67cd1f068b9826d6ee1529c653dbf90fa542ef8f2636ae" + }, + { + "name": "1.7B-8-conv_in", + "rows": 1, + "bytes": 12288, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "7e9ceb9220c05a6b5069aee118ad7bff5449da2d9e91c9ff8c5eea17cf7263e8" + }, + { + "name": "1.7B-8-conv_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "f1469364d1355c14c380655d42e4aef357c54ee58a03f767e65073c69bcc77db" + }, + { + "name": "1.7B-8-conv_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "347e2cee0ad33b3cf957d1487e3fd2200eed2b6f4de206da8f2b0758ab471c86" + }, + { + "name": "1.7B-8-ffn_gate", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "3359e636ffbda8f5c9a3267435991cca16cc88e19b4919f566a4c3d3a96a90cd" + }, + { + "name": "1.7B-8-ffn_gate", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "5d7265f9038c45a2a7414e9bf751a2cdee8ead2da6d0144ef8bc73500083f6f1" + }, + { + "name": "1.7B-8-ffn_up", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "6c9c205deb589766242f9176c5d5cb06b5f114ae4c0144c4b168c16fed380ace" + }, + { + "name": "1.7B-8-ffn_up", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "44147335dcb02a2216a2d116959141df0a9aad2bec658fae007ae864ea0ec157" + }, + { + "name": "1.7B-8-ffn_down", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "c9535b39ba4ef83911282db1f28fca22373b318d7749ac18c90ca0a91a236bde" + }, + { + "name": "1.7B-8-ffn_down", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "8555da8c544587c8042e0bf321c6288974d6f2e58eabf4079966ed53504e8510" + }, + { + "name": "1.7B-8-attention_q", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "cfeac51b5f537a6e2bcc4c466a9912a06d2b4c0d4d0e5418f04678bc99febb76" + }, + { + "name": "1.7B-8-attention_q", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "c12320e5ac155f07a393be2542fff7995eccd912097c79f8ac63aa3dadf750d0" + }, + { + "name": "1.7B-8-attention_k", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "0d64a33538106896db23021a7251d26e7622f90e950561d15f0d041b70ce4802" + }, + { + "name": "1.7B-8-attention_k", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "bd2d6913ec4438c993dcebe577870b403caae745a38a7af3c9fe7ea2c849230e" + }, + { + "name": "1.7B-8-attention_v", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "8bbf123e32dd083bd8515984bd3f5d24e8520e68366cf5cbff99f6c42f9e9bbd" + }, + { + "name": "1.7B-8-attention_v", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "5672c89b1209a47f4db84bbbed85ad5677542e28f1f4bcdbc95098d2d1027aa0" + }, + { + "name": "1.7B-8-attention_gate", + "rows": 64, + "bytes": 2048, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "9d5e0d6dc77e56482a4652be9b53cdecfd809c91c6daf9cd86d598cf6ea5515d" + }, + { + "name": "1.7B-8-attention_gate", + "rows": 1, + "bytes": 32, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "490faa9e36e8cd958d4d64e8f85a708191d50e1c8b402c5686e8bdeefa97d7ae" + }, + { + "name": "1.7B-8-attention_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "9309a2af2e0a4d428c0c5dc0cea2732f468c93aaff21a7de62689c7c949b445f" + }, + { + "name": "1.7B-8-attention_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "9aee14bdf4e46e44048a96d2ecce55556f03067b17f970bb6b693c6d3c6cb9fd" + }, + { + "name": "1.7B-8-vocabulary_head", + "rows": 64, + "bytes": 16384000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "60e65041ceddf857ed5d3f834ee2e3e9031aa35657546522c649c229cc69cadd" + }, + { + "name": "1.7B-8-vocabulary_head", + "rows": 1, + "bytes": 256000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "0eb8a7d5cff089e99330ff2b8de752bcbde52172e7a80d47f97553714027587c" + }, + { + "name": "VL-2B-4-conv_in", + "rows": 64, + "bytes": 786432, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "d14b70a308f20d5218b334611a93b842b44e8608c806f4e7e7fffc0ad14a9e9f" + }, + { + "name": "VL-2B-4-conv_in", + "rows": 1, + "bytes": 12288, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "719399211617079e28556f60b5da3216dc841a0c5f40f51223f210e081c14003" + }, + { + "name": "VL-2B-4-conv_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "a781c7e21bd0ed2ed093e11d769fa51868182e8a96a8ccaf8006527cf768a1cb" + }, + { + "name": "VL-2B-4-conv_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "8bc576734bd022af17e8c1d5f6e0e38088c42165aa06e297822024a3762dd806" + }, + { + "name": "VL-2B-4-ffn_gate", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "4f23d79f8e70a5da00d1b6c7369456881ce93f202bd7677c731a42b8bf13afa7" + }, + { + "name": "VL-2B-4-ffn_gate", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "fc3260f84dd8a6cd21baa4e4690c37e26e2a09b75f27bdccfc0b48e1a4e155a2" + }, + { + "name": "VL-2B-4-ffn_up", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "50a86965edc35a4281b42ad4574c34de38cf90ae70309a694c54f6fc9033b813" + }, + { + "name": "VL-2B-4-ffn_up", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "14890b0147213d78a0d31c91dbdb82efbef19a820cc80d281b37d0441d2963b5" + }, + { + "name": "VL-2B-4-ffn_down", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "10283fdeee7e16d060a6e045bb7f91c25dadc1cdc0251176a658a9b4c2126733" + }, + { + "name": "VL-2B-4-ffn_down", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "98c725a270344e983bedaeaba0de68165a5ff08404ca24012c943835d7a41b4d" + }, + { + "name": "VL-2B-4-attention_q", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "ccdfc7269632f8e52d30bbc34c55535b7fce5f67ad555f75a4b40f4ed77ebf03" + }, + { + "name": "VL-2B-4-attention_q", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "80c2bb98cc9414ddeb8708afe894f12a1c78380124bcf65b261af28d5afba456" + }, + { + "name": "VL-2B-4-attention_k", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "6c46ee9e09a66308ad24f7fcc48c806f5066f11b77cad9612cd356ffe4634728" + }, + { + "name": "VL-2B-4-attention_k", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "1b51d5ed6ef29f58baa2788eafaa855f89bb12320babdc361f2cd9f58816ac5b" + }, + { + "name": "VL-2B-4-attention_v", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "b36ddb06295061232d83d792cf1d992f88aff3c9394bb745575bfa07e66ffdb4" + }, + { + "name": "VL-2B-4-attention_v", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "e46f37723d8adf961c8e8e62cb1d09d74a33fdd36562fb52bb7f804aeccb129c" + }, + { + "name": "VL-2B-4-attention_gate", + "rows": 64, + "bytes": 2048, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "a18b3d969a2b0b1ded99fc9bdf947a780f5c8b711ea1c57a893bb2d9f2dcb810" + }, + { + "name": "VL-2B-4-attention_gate", + "rows": 1, + "bytes": 32, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "3137099219d5c4ecf39915438ad8f89e506b5b84b2898fa3dcd4b0306f33f77d" + }, + { + "name": "VL-2B-4-attention_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "193973f3a4fc797a29993f8a8c52b3659f13a2a044740cc8061011a9e120bcf8" + }, + { + "name": "VL-2B-4-attention_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "2f9d4462c35d055eb24a15602a150770bd2387229cdd4ae7a028b26a4dc86927" + }, + { + "name": "VL-2B-4-vocabulary_head", + "rows": 64, + "bytes": 16384000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "64bd27b14f896401645b37e539c37ae3d35d28c1df24bc48f6a4c18fabf78a4c" + }, + { + "name": "VL-2B-4-vocabulary_head", + "rows": 1, + "bytes": 256000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "95b91cd0681074c14320eaa82430d3ddd9d21a5015e726c98c797b96206ffd0e" + }, + { + "name": "VL-2B-6-conv_in", + "rows": 64, + "bytes": 786432, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "767f316b194381f15ad6ecff5a88bb2764cf709d4e1ef2d841e32a52cc960ad3" + }, + { + "name": "VL-2B-6-conv_in", + "rows": 1, + "bytes": 12288, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "f747092d8186ca673cd682c56a429eaf2c27631c02b9903ba803532865043db5" + }, + { + "name": "VL-2B-6-conv_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "e133c7b157ae2341d7c533a153789ed4c8ff5fe93df6146d57cf855ee81f5d93" + }, + { + "name": "VL-2B-6-conv_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "23087af72979d3a2dc227423c1b8c31d4ff5954224f07798b82dd730c494570a" + }, + { + "name": "VL-2B-6-ffn_gate", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "5e22fb9cad2fbf1de6f75a4bd152691c72fec6c5daffab2da265ed56e0d31fcd" + }, + { + "name": "VL-2B-6-ffn_gate", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "2d240b08a3628383264bb75eb0b522693059025db8cadb554d77b493703d0e9f" + }, + { + "name": "VL-2B-6-ffn_up", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "db0fa182e03793b1f02d4b297035039cf6af4001d0bb3f70cef2674b15eb4270" + }, + { + "name": "VL-2B-6-ffn_up", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "4ff805d85a0816552f0774749cd18aa1daac3e67e74345b907331358fca800a8" + }, + { + "name": "VL-2B-6-ffn_down", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "2daecfa2bf09af4963cbc61d1fc1cd8c187d12b7f94ad5e3faa7bbdfcf164c52" + }, + { + "name": "VL-2B-6-ffn_down", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "531988206cfc6f52e7e6b3c227132392bf3204b87a84f422bcd5bb80c162d68a" + }, + { + "name": "VL-2B-6-attention_q", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "7dd67e902a2eef702d22c1f1ea921c09e4ee7d054d66e5c7d9e2cbb7ecaa31fb" + }, + { + "name": "VL-2B-6-attention_q", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "731b98f325784057f48d09e9eb0e5b87f1c2e4e1089729873689fea506272ca7" + }, + { + "name": "VL-2B-6-attention_k", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "ec9d623b94b96e7f16efd8b78fbeb6594daef7712d34101e8171efb9d9a07e76" + }, + { + "name": "VL-2B-6-attention_k", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "897083fbf229dca38a8de1d30fac9a6098d55ec30874d2ba14a692d34bddb436" + }, + { + "name": "VL-2B-6-attention_v", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "c9767157863db3b698250112aedfbf14601174a6e1bde137f72881df5f50c1a3" + }, + { + "name": "VL-2B-6-attention_v", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "a8d94974c7616852ac893dd29ba55aa7bf8e82c9760dd7793157eb96df59358f" + }, + { + "name": "VL-2B-6-attention_gate", + "rows": 64, + "bytes": 2048, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "ae7f4042b17c5c7b77562de99ec66e581e5815e9f2e61d8cec0c461d7d0eb2cb" + }, + { + "name": "VL-2B-6-attention_gate", + "rows": 1, + "bytes": 32, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "0e7e3e65904ebd568f7b50e3856311cd08f4ce5bd2b5c25e7cccaeb04bf1d2ad" + }, + { + "name": "VL-2B-6-attention_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "df60fd6d1502456327eda9c18e9a1c8f083f57f168347c50ed13cbf327386c92" + }, + { + "name": "VL-2B-6-attention_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "12f97abed04460ce87b0fedebcae80babd945ccb9cde3de78e704f17cb224e9e" + }, + { + "name": "VL-2B-6-vocabulary_head", + "rows": 64, + "bytes": 16384000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "a2cdac531945e51aa61719171460f3e40cf8b658f6f4532dd46fa944b10483ad" + }, + { + "name": "VL-2B-6-vocabulary_head", + "rows": 1, + "bytes": 256000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "3af09e6edb5b63ee9acb212a1fe4f23d82d7dfdf39ae63617c6fb60ebc523362" + }, + { + "name": "VL-2B-8-conv_in", + "rows": 64, + "bytes": 786432, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "71bf9b014f1cc1b8edc7167067e790c3a5e536a88109751b5ed4f09528098597" + }, + { + "name": "VL-2B-8-conv_in", + "rows": 1, + "bytes": 12288, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "abe2d50106ca7025d34167f9b593f034e3cccfd56c2aa5473efa2505b6469c9a" + }, + { + "name": "VL-2B-8-conv_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "3a88ff8d8fb1b7d99092f109dedf5f81ca6d76c32433c9b6aeb3596b7f6ab8b2" + }, + { + "name": "VL-2B-8-conv_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "d32cc9511e429d97a0c492a42d9b7b6184ad6165ac1ed9e3b355ed6765af85fc" + }, + { + "name": "VL-2B-8-ffn_gate", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "e0b4f14b9e9b91516bd391a06838292f5ff79ae9cbc429fe404bde9f5a130e08" + }, + { + "name": "VL-2B-8-ffn_gate", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "f5caab73d84c4262377769654faaba8c285b694455e329264919a3f9641db7ef" + }, + { + "name": "VL-2B-8-ffn_up", + "rows": 64, + "bytes": 983040, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "fcce4ac31aa0f9ef38b3cd0cb7c25398dcf6ed68a309362418b5814af62eca3e" + }, + { + "name": "VL-2B-8-ffn_up", + "rows": 1, + "bytes": 15360, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "112d5b19e6a549b3ea21198dc4f933dbaeeb6f3579cc895c076c6b7f406cd105" + }, + { + "name": "VL-2B-8-ffn_down", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "6896f97343dfd72dfcdb74d4f1f5141bd4283249844922b8def07210c5abf213" + }, + { + "name": "VL-2B-8-ffn_down", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "6fa80d2c836c95a713b9a496e3d75954b69f48c775fe975f6011f3a9583f0870" + }, + { + "name": "VL-2B-8-attention_q", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "2406496d7aac10e40363859bcbe52cdcbbd424d233cedd915f90ea89bbc351f4" + }, + { + "name": "VL-2B-8-attention_q", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "7ebc981eadeeb63851ef1ac62366369946d4cd85699291ef54ea48ae45b9ed9c" + }, + { + "name": "VL-2B-8-attention_k", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "e04f2457b18a1378260dcc1ab3f6372c8f82a54c65b95ca3b95575fc36798874" + }, + { + "name": "VL-2B-8-attention_k", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "ca775c01620b1478372d4823c64a976b5e0ef3ad0d3f1c4d90e731cc4f8671a9" + }, + { + "name": "VL-2B-8-attention_v", + "rows": 64, + "bytes": 65536, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "3624b048e8946768c2f095071dd36e53475847d188b87da9146ae6d9fe064040" + }, + { + "name": "VL-2B-8-attention_v", + "rows": 1, + "bytes": 1024, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "861913b23c3fcbafe53476e6537fe0c03da18bb338a6d50842dd13f7e84376e7" + }, + { + "name": "VL-2B-8-attention_gate", + "rows": 64, + "bytes": 2048, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "5ac650fc043aaa905783be602dfbddbc6a8f0c0a94d78d7188743bc70801508e" + }, + { + "name": "VL-2B-8-attention_gate", + "rows": 1, + "bytes": 32, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "95cbe3c3772e06917f9b2aa90cafd9ffd5cbeacf97a131ed3a77862627308818" + }, + { + "name": "VL-2B-8-attention_out", + "rows": 64, + "bytes": 262144, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "0c026f593adfcb6b3ed08c37bebbfc5d0cfc265eb1e6bf4f2b3dd3af8d3de347" + }, + { + "name": "VL-2B-8-attention_out", + "rows": 1, + "bytes": 4096, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "e5632e43d6d113a76edddfecc4f162eacc1c6c407b8dcf58b35487528da6e1e1" + }, + { + "name": "VL-2B-8-vocabulary_head", + "rows": 64, + "bytes": 16384000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "61f3c553b0a0daf33f2dd29f46677112dae2639fd338014b7ebdd22558c6b1cb" + }, + { + "name": "VL-2B-8-vocabulary_head", + "rows": 1, + "bytes": 256000, + "equals_prior_FP64_exact_MMA_capture": true, + "sha256": "e250665cdfd5e271e66e762ae85ec1bae526ef802cc62ec9f736be9d52622f13" + } + ], + "operator_contract": { + "m4": "SUMMARY 6 kernels: 6 pass gates 1-4, 0 fail, 0 known findings; 2 controls: 2 red as planted, 0 missed; 0 limit problems; gates 10718 ms, limits 147 ms", + "m5": "SUMMARY 6 kernels: 6 pass gates 1-4, 0 fail, 0 known findings; 2 controls: 2 red as planted, 0 missed; 0 limit problems; gates 8872 ms, limits 88 ms" + }, + "native_project_sha256": "184f7fd1f7e1794f211f6ca0c5c6c6dcd8a2930a883565a16812fe33ddf7c233", + "sources_sha256": { + "zig/kernels/metal/core/binned_projection.metal": "8119aa792197ebf3c9b12f767fe0dfe12482189340be8e75ae93f7910ac944ca", + "zig/src/core/binned_projection.zig": "14d6a42a263d639435eae208a6e808830dfb43e86a6bbfad3c67d4499af85d4f" + } +} diff --git a/docs/receipts/lfm3/cold-prefill-ab.json b/docs/receipts/lfm3/cold-prefill-ab.json new file mode 100644 index 0000000000..2b636885db --- /dev/null +++ b/docs/receipts/lfm3/cold-prefill-ab.json @@ -0,0 +1,383 @@ +{ + "baseline_native_sha256": "a8df9730ee637237545311a6753dd4bdcb13e860700d65a01a033ae818ee0d18", + "rowaxis_native_sha256": "5c8a04d4af599879797f4f5bed8e00ab7d9d70732e698bd2a1ec2a01ec762e09", + "completed_matching_pairs": 16, + "cells": [ + { + "family": "text", + "prompt_tokens": 2048, + "requested_pairs": 3, + "completed_pairs": 3, + "before_prefill_median_s": 1.239529791, + "after_prefill_median_s": 0.812958542, + "speed_ratio": 1.524714640368463, + "phases": { + "before": [ + { + "rep": 0, + "prefill_s": 1.239792167, + "ttft_s": 1.2504969998262823, + "token_sha256": "c244b715e827fd1ce59aa4e249cb47900812abeee1482b0a79a1b251fe0f3262", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 1491863256 + }, + { + "rep": 1, + "prefill_s": 1.238933042, + "ttft_s": 1.2502966253086925, + "token_sha256": "c244b715e827fd1ce59aa4e249cb47900812abeee1482b0a79a1b251fe0f3262", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 1491846848 + }, + { + "rep": 2, + "prefill_s": 1.239529791, + "ttft_s": 1.2511707078665495, + "token_sha256": "c244b715e827fd1ce59aa4e249cb47900812abeee1482b0a79a1b251fe0f3262", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 1491830440 + } + ], + "after": [ + { + "rep": 0, + "prefill_s": 0.81294775, + "ttft_s": 0.8246144163422287, + "token_sha256": "c244b715e827fd1ce59aa4e249cb47900812abeee1482b0a79a1b251fe0f3262", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 1490699896 + }, + { + "rep": 1, + "prefill_s": 0.812958542, + "ttft_s": 0.8243804168887436, + "token_sha256": "c244b715e827fd1ce59aa4e249cb47900812abeee1482b0a79a1b251fe0f3262", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 1490667128 + }, + { + "rep": 2, + "prefill_s": 0.813141375, + "ttft_s": 0.8247690829448402, + "token_sha256": "c244b715e827fd1ce59aa4e249cb47900812abeee1482b0a79a1b251fe0f3262", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 1490699896 + } + ] + } + }, + { + "family": "text", + "prompt_tokens": 8192, + "requested_pairs": 3, + "completed_pairs": 3, + "before_prefill_median_s": 5.307524, + "after_prefill_median_s": 3.608894709, + "speed_ratio": 1.4706785395439477, + "phases": { + "before": [ + { + "rep": 0, + "prefill_s": 5.306101708, + "ttft_s": 5.318894917145371, + "token_sha256": "3f913af21b98f7168ba1fcd27bd9f16b62982dcf6168b991091bbd6e4d9e7fed", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 1630586536 + }, + { + "rep": 1, + "prefill_s": 5.307524, + "ttft_s": 5.319628749974072, + "token_sha256": "3f913af21b98f7168ba1fcd27bd9f16b62982dcf6168b991091bbd6e4d9e7fed", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 1630570152 + }, + { + "rep": 2, + "prefill_s": 5.31343825, + "ttft_s": 5.325319999828935, + "token_sha256": "3f913af21b98f7168ba1fcd27bd9f16b62982dcf6168b991091bbd6e4d9e7fed", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 1630652120 + } + ], + "after": [ + { + "rep": 0, + "prefill_s": 3.608894709, + "ttft_s": 3.621469959151, + "token_sha256": "3f913af21b98f7168ba1fcd27bd9f16b62982dcf6168b991091bbd6e4d9e7fed", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 1629488736 + }, + { + "rep": 1, + "prefill_s": 3.610029625, + "ttft_s": 3.6229068338871, + "token_sha256": "3f913af21b98f7168ba1fcd27bd9f16b62982dcf6168b991091bbd6e4d9e7fed", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 1629570752 + }, + { + "rep": 2, + "prefill_s": 3.607758625, + "ttft_s": 3.620500334072858, + "token_sha256": "3f913af21b98f7168ba1fcd27bd9f16b62982dcf6168b991091bbd6e4d9e7fed", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 1629488784 + } + ] + } + }, + { + "family": "text", + "prompt_tokens": 32768, + "requested_pairs": 1, + "completed_pairs": 1, + "before_prefill_median_s": 27.314690833, + "after_prefill_median_s": 20.488389292, + "speed_ratio": 1.333179023675884, + "phases": { + "before": [ + { + "rep": 0, + "prefill_s": 27.314690833, + "ttft_s": 27.331630500033498, + "token_sha256": "e15e7490e640d938a0a846dfc9de1dd7a51e50ae97ca4fd42a213cbb633f66f5", + "body_sha256": "b11cb39bb728d97c62594c8a4f2004846f06aeddebda9e4c825136a31448bcb6", + "peak_physical_footprint_bytes": 2185791144 + } + ], + "after": [ + { + "rep": 0, + "prefill_s": 20.488389292, + "ttft_s": 20.504653749987483, + "token_sha256": "e15e7490e640d938a0a846dfc9de1dd7a51e50ae97ca4fd42a213cbb633f66f5", + "body_sha256": "b11cb39bb728d97c62594c8a4f2004846f06aeddebda9e4c825136a31448bcb6", + "peak_physical_footprint_bytes": 2184644240 + } + ] + } + }, + { + "family": "text", + "prompt_tokens": 65536, + "requested_pairs": 1, + "completed_pairs": 1, + "before_prefill_median_s": 73.402432042, + "after_prefill_median_s": 61.632932625, + "speed_ratio": 1.1909612104394003, + "phases": { + "before": [ + { + "rep": 0, + "prefill_s": 73.402432042, + "ttft_s": 73.4243381661363, + "token_sha256": "cf6c4062b60dc49e0ca503b7be2aaf3d247d73be037d269cf73e127a5e35aa32", + "body_sha256": "3aa9a0a57dfaccac1354b37af6015a319cb24fce9c0fb30f4b6ddb3f12a48639", + "peak_physical_footprint_bytes": 2926167768 + } + ], + "after": [ + { + "rep": 0, + "prefill_s": 61.632932625, + "ttft_s": 61.653554582968354, + "token_sha256": "cf6c4062b60dc49e0ca503b7be2aaf3d247d73be037d269cf73e127a5e35aa32", + "body_sha256": "3aa9a0a57dfaccac1354b37af6015a319cb24fce9c0fb30f4b6ddb3f12a48639", + "peak_physical_footprint_bytes": 2925315776 + } + ] + } + }, + { + "family": "vl", + "prompt_tokens": 2048, + "requested_pairs": 3, + "completed_pairs": 3, + "before_prefill_median_s": 1.2618445, + "after_prefill_median_s": 0.815432416, + "speed_ratio": 1.5474544244755655, + "phases": { + "before": [ + { + "rep": 0, + "prefill_s": 1.261758042, + "ttft_s": 1.273614791687578, + "token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 2151958712 + }, + { + "rep": 1, + "prefill_s": 1.2618445, + "ttft_s": 1.2731465827673674, + "token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 2151925920 + }, + { + "rep": 2, + "prefill_s": 1.26217825, + "ttft_s": 1.2739000832661986, + "token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 2151893128 + } + ], + "after": [ + { + "rep": 0, + "prefill_s": 0.815139, + "ttft_s": 0.8269048747606575, + "token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 2149976200 + }, + { + "rep": 1, + "prefill_s": 0.815432416, + "ttft_s": 0.8269431251101196, + "token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 2149976200 + }, + { + "rep": 2, + "prefill_s": 0.816996041, + "ttft_s": 0.8286756249144673, + "token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89", + "body_sha256": "b0cbef400cfcf380f0fe40ce95d246490b56d875bf7a9de9362a59d0670b2c81", + "peak_physical_footprint_bytes": 2150041736 + } + ] + } + }, + { + "family": "vl", + "prompt_tokens": 8192, + "requested_pairs": 3, + "completed_pairs": 3, + "before_prefill_median_s": 5.402433084, + "after_prefill_median_s": 3.61744925, + "speed_ratio": 1.4934371460774467, + "phases": { + "before": [ + { + "rep": 0, + "prefill_s": 5.399245833, + "ttft_s": 5.410978707950562, + "token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 2290616432 + }, + { + "rep": 1, + "prefill_s": 5.403565458, + "ttft_s": 5.416440249886364, + "token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 2290583664 + }, + { + "rep": 2, + "prefill_s": 5.402433084, + "ttft_s": 5.415588291827589, + "token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 2290583664 + } + ], + "after": [ + { + "rep": 0, + "prefill_s": 3.61744925, + "ttft_s": 3.630715959239751, + "token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 2288797856 + }, + { + "rep": 1, + "prefill_s": 3.624671708, + "ttft_s": 3.6372522921301425, + "token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 2288797856 + }, + { + "rep": 2, + "prefill_s": 3.608385375, + "ttft_s": 3.6207989170216024, + "token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0", + "body_sha256": "3f4418924b795e4b3cf6cdbe448c7e3f51449ee6a818577eee336e38e3ee182c", + "peak_physical_footprint_bytes": 2288748680 + } + ] + } + }, + { + "family": "vl", + "prompt_tokens": 32768, + "requested_pairs": 1, + "completed_pairs": 1, + "before_prefill_median_s": 27.686642834, + "after_prefill_median_s": 20.536701917, + "speed_ratio": 1.3481542920521907, + "phases": { + "before": [ + { + "rep": 0, + "prefill_s": 27.686642834, + "ttft_s": 27.70385383302346, + "token_sha256": "f35dfbfd86653608bd3aa0fe6672229ecfe9ec17bf45ebcdb91dc7c7e894af4a", + "body_sha256": "b11cb39bb728d97c62594c8a4f2004846f06aeddebda9e4c825136a31448bcb6", + "peak_physical_footprint_bytes": 2845788224 + } + ], + "after": [ + { + "rep": 0, + "prefill_s": 20.536701917, + "ttft_s": 20.55388933280483, + "token_sha256": "f35dfbfd86653608bd3aa0fe6672229ecfe9ec17bf45ebcdb91dc7c7e894af4a", + "body_sha256": "b11cb39bb728d97c62594c8a4f2004846f06aeddebda9e4c825136a31448bcb6", + "peak_physical_footprint_bytes": 2843887776 + } + ] + } + }, + { + "family": "vl", + "prompt_tokens": 65536, + "requested_pairs": 1, + "completed_pairs": 1, + "before_prefill_median_s": 73.920498208, + "after_prefill_median_s": 59.64693, + "speed_ratio": 1.2393009700247775, + "phases": { + "before": [ + { + "rep": 0, + "prefill_s": 73.920498208, + "ttft_s": 73.94245912507176, + "token_sha256": "6dc17e16c1bdd265cbfecc43f466db1fb29c9240e7818d912ecc773b568b4856", + "body_sha256": "3aa9a0a57dfaccac1354b37af6015a319cb24fce9c0fb30f4b6ddb3f12a48639", + "peak_physical_footprint_bytes": 3586115720 + } + ], + "after": [ + { + "rep": 0, + "prefill_s": 59.64693, + "ttft_s": 59.66807400016114, + "token_sha256": "6dc17e16c1bdd265cbfecc43f466db1fb29c9240e7818d912ecc773b568b4856", + "body_sha256": "3aa9a0a57dfaccac1354b37af6015a319cb24fce9c0fb30f4b6ddb3f12a48639", + "peak_physical_footprint_bytes": 3584559264 + } + ] + } + } + ] +} diff --git a/docs/receipts/lfm3/cold-prompt-sha256.json b/docs/receipts/lfm3/cold-prompt-sha256.json new file mode 100644 index 0000000000..9398e0ac47 --- /dev/null +++ b/docs/receipts/lfm3/cold-prompt-sha256.json @@ -0,0 +1,18 @@ +{ + "text-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "text-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "text-32768.body.json": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "text-32768.ids.json": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "text-65536.body.json": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "text-65536.ids.json": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "text-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "text-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd", + "vl-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "vl-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "vl-32768.body.json": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "vl-32768.ids.json": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "vl-65536.body.json": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "vl-65536.ids.json": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "vl-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "vl-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd" +} diff --git a/docs/receipts/lfm3/cold-prompts/text-2048.body.json b/docs/receipts/lfm3/cold-prompts/text-2048.body.json new file mode 100644 index 0000000000..b3cd9a67e0 --- /dev/null +++ b/docs/receipts/lfm3/cold-prompts/text-2048.body.json @@ -0,0 +1 @@ +{"messages": [{"role": "user", "content": "A public benchmark fixture. Explain this sequence.\n alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha"}], "max_tokens": 16, "temperature": 0, "top_p": 1, "top_k": 0, "repetition_penalty": 1, "stream": true, "stream_options": {"include_usage": true}, "enable_thinking": false, "chat_template_kwargs": {"enable_thinking": false}} diff --git a/docs/receipts/lfm3/cold-prompts/text-2048.ids.json b/docs/receipts/lfm3/cold-prompts/text-2048.ids.json new file mode 100644 index 0000000000..9a7a1f3c30 --- /dev/null +++ b/docs/receipts/lfm3/cold-prompts/text-2048.ids.json @@ -0,0 +1 @@ +[124894, 124899, 5922, 207, 41, 1251, 40524, 51639, 22, 42847, 532, 8055, 318, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 124900, 207, 124899, 63514, 207, 124901, 124902] diff --git a/docs/receipts/lfm3/cold-prompts/text-32768.body.json b/docs/receipts/lfm3/cold-prompts/text-32768.body.json new file mode 100644 index 0000000000..373db8d0fb --- /dev/null +++ b/docs/receipts/lfm3/cold-prompts/text-32768.body.json @@ -0,0 +1 @@ +{"messages": [{"role": "user", "content": "A public benchmark fixture. Explain this sequence.\n alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha"}], "max_tokens": 16, "temperature": 0, "top_p": 1, "top_k": 0, "repetition_penalty": 1, "stream": true, "stream_options": {"include_usage": true}, "enable_thinking": false, "chat_template_kwargs": {"enable_thinking": false}} diff --git a/docs/receipts/lfm3/cold-prompts/text-32768.ids.json b/docs/receipts/lfm3/cold-prompts/text-32768.ids.json new file mode 100644 index 0000000000..1377e52524 --- /dev/null +++ b/docs/receipts/lfm3/cold-prompts/text-32768.ids.json @@ -0,0 +1 @@ +[124894, 124899, 5922, 207, 41, 1251, 40524, 51639, 22, 42847, 532, 8055, 318, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 124900, 207, 124899, 63514, 207, 124901, 124902] diff --git a/docs/receipts/lfm3/cold-prompts/text-65536.body.json b/docs/receipts/lfm3/cold-prompts/text-65536.body.json new file mode 100644 index 0000000000..a5aceef332 --- /dev/null +++ b/docs/receipts/lfm3/cold-prompts/text-65536.body.json @@ -0,0 +1 @@ +{"messages": [{"role": "user", "content": "A public benchmark fixture. Explain this sequence.\n alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha"}], "max_tokens": 16, "temperature": 0, "top_p": 1, "top_k": 0, "repetition_penalty": 1, "stream": true, "stream_options": {"include_usage": true}, "enable_thinking": false, "chat_template_kwargs": {"enable_thinking": false}} diff --git a/docs/receipts/lfm3/cold-prompts/text-65536.ids.json b/docs/receipts/lfm3/cold-prompts/text-65536.ids.json new file mode 100644 index 0000000000..7bb64d0b1d --- /dev/null +++ b/docs/receipts/lfm3/cold-prompts/text-65536.ids.json @@ -0,0 +1 @@ +[124894, 124899, 5922, 207, 41, 1251, 40524, 51639, 22, 42847, 532, 8055, 318, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 124900, 207, 124899, 63514, 207, 124901, 124902] diff --git a/docs/receipts/lfm3/cold-prompts/text-8192.body.json b/docs/receipts/lfm3/cold-prompts/text-8192.body.json new file mode 100644 index 0000000000..87728222f6 --- /dev/null +++ b/docs/receipts/lfm3/cold-prompts/text-8192.body.json @@ -0,0 +1 @@ +{"messages": [{"role": "user", "content": "A public benchmark fixture. Explain this sequence.\n alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha"}], "max_tokens": 16, "temperature": 0, "top_p": 1, "top_k": 0, "repetition_penalty": 1, "stream": true, "stream_options": {"include_usage": true}, "enable_thinking": false, "chat_template_kwargs": {"enable_thinking": false}} diff --git a/docs/receipts/lfm3/cold-prompts/text-8192.ids.json b/docs/receipts/lfm3/cold-prompts/text-8192.ids.json new file mode 100644 index 0000000000..9e622e081e --- /dev/null +++ b/docs/receipts/lfm3/cold-prompts/text-8192.ids.json @@ -0,0 +1 @@ +[124894, 124899, 5922, 207, 41, 1251, 40524, 51639, 22, 42847, 532, 8055, 318, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 124900, 207, 124899, 63514, 207, 124901, 124902] diff --git a/docs/receipts/lfm3/cold-prompts/vl-2048.body.json b/docs/receipts/lfm3/cold-prompts/vl-2048.body.json new file mode 100644 index 0000000000..b3cd9a67e0 --- /dev/null +++ b/docs/receipts/lfm3/cold-prompts/vl-2048.body.json @@ -0,0 +1 @@ +{"messages": [{"role": "user", "content": "A public benchmark fixture. Explain this sequence.\n alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha"}], "max_tokens": 16, "temperature": 0, "top_p": 1, "top_k": 0, "repetition_penalty": 1, "stream": true, "stream_options": {"include_usage": true}, "enable_thinking": false, "chat_template_kwargs": {"enable_thinking": false}} diff --git a/docs/receipts/lfm3/cold-prompts/vl-2048.ids.json b/docs/receipts/lfm3/cold-prompts/vl-2048.ids.json new file mode 100644 index 0000000000..9a7a1f3c30 --- /dev/null +++ b/docs/receipts/lfm3/cold-prompts/vl-2048.ids.json @@ -0,0 +1 @@ +[124894, 124899, 5922, 207, 41, 1251, 40524, 51639, 22, 42847, 532, 8055, 318, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 124900, 207, 124899, 63514, 207, 124901, 124902] diff --git a/docs/receipts/lfm3/cold-prompts/vl-32768.body.json b/docs/receipts/lfm3/cold-prompts/vl-32768.body.json new file mode 100644 index 0000000000..373db8d0fb --- /dev/null +++ b/docs/receipts/lfm3/cold-prompts/vl-32768.body.json @@ -0,0 +1 @@ +{"messages": [{"role": "user", "content": "A public benchmark fixture. Explain this sequence.\n alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha"}], "max_tokens": 16, "temperature": 0, "top_p": 1, "top_k": 0, "repetition_penalty": 1, "stream": true, "stream_options": {"include_usage": true}, "enable_thinking": false, "chat_template_kwargs": {"enable_thinking": false}} diff --git a/docs/receipts/lfm3/cold-prompts/vl-32768.ids.json b/docs/receipts/lfm3/cold-prompts/vl-32768.ids.json new file mode 100644 index 0000000000..1377e52524 --- /dev/null +++ b/docs/receipts/lfm3/cold-prompts/vl-32768.ids.json @@ -0,0 +1 @@ +[124894, 124899, 5922, 207, 41, 1251, 40524, 51639, 22, 42847, 532, 8055, 318, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 124900, 207, 124899, 63514, 207, 124901, 124902] diff --git a/docs/receipts/lfm3/cold-prompts/vl-65536.body.json b/docs/receipts/lfm3/cold-prompts/vl-65536.body.json new file mode 100644 index 0000000000..a5aceef332 --- /dev/null +++ b/docs/receipts/lfm3/cold-prompts/vl-65536.body.json @@ -0,0 +1 @@ +{"messages": [{"role": "user", "content": "A public benchmark fixture. Explain this sequence.\n alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha"}], "max_tokens": 16, "temperature": 0, "top_p": 1, "top_k": 0, "repetition_penalty": 1, "stream": true, "stream_options": {"include_usage": true}, "enable_thinking": false, "chat_template_kwargs": {"enable_thinking": false}} diff --git a/docs/receipts/lfm3/cold-prompts/vl-65536.ids.json b/docs/receipts/lfm3/cold-prompts/vl-65536.ids.json new file mode 100644 index 0000000000..7bb64d0b1d --- /dev/null +++ b/docs/receipts/lfm3/cold-prompts/vl-65536.ids.json @@ -0,0 +1 @@ +[124894, 124899, 5922, 207, 41, 1251, 40524, 51639, 22, 42847, 532, 8055, 318, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 124900, 207, 124899, 63514, 207, 124901, 124902] diff --git a/docs/receipts/lfm3/cold-prompts/vl-8192.body.json b/docs/receipts/lfm3/cold-prompts/vl-8192.body.json new file mode 100644 index 0000000000..87728222f6 --- /dev/null +++ b/docs/receipts/lfm3/cold-prompts/vl-8192.body.json @@ -0,0 +1 @@ +{"messages": [{"role": "user", "content": "A public benchmark fixture. Explain this sequence.\n alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha"}], "max_tokens": 16, "temperature": 0, "top_p": 1, "top_k": 0, "repetition_penalty": 1, "stream": true, "stream_options": {"include_usage": true}, "enable_thinking": false, "chat_template_kwargs": {"enable_thinking": false}} diff --git a/docs/receipts/lfm3/cold-prompts/vl-8192.ids.json b/docs/receipts/lfm3/cold-prompts/vl-8192.ids.json new file mode 100644 index 0000000000..9e622e081e --- /dev/null +++ b/docs/receipts/lfm3/cold-prompts/vl-8192.ids.json @@ -0,0 +1 @@ +[124894, 124899, 5922, 207, 41, 1251, 40524, 51639, 22, 42847, 532, 8055, 318, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 23018, 124900, 207, 124899, 63514, 207, 124901, 124902] diff --git a/docs/receipts/lfm3/current-public-long-context.json b/docs/receipts/lfm3/current-public-long-context.json new file mode 100644 index 0000000000..7b01d10e61 --- /dev/null +++ b/docs/receipts/lfm3/current-public-long-context.json @@ -0,0 +1,956 @@ +{ + "schema_version": 1, + "scope": "Current public runtime cold long-context HTTP checks. Local48GiB M4 covers text1.7B4bit and VL2B4bit with TEXT prompts; no images in this context experiment.", + "status": "All measured context checks passed; no speed or FP64 fidelity acceptance claimed.", + "runtime_fix_commit": "dd7263c26", + "native_binary_sha256": "3d4a3daeda697420bfbba6d81972896c2bf9e31efaf6f12321ee8487a299354b", + "runs": [ + { + "host": { + "gpu": "Apple M4 Pro", + "gpu_cores": 20, + "physical_memory_bytes": 51539607552, + "os": "ProductName:\t\tmacOS\nProductVersion:\t\t26.5.1\nBuildVersion:\t\t25F80\n" + }, + "status": { + "actual_exit": 0, + "requests_completed": 8, + "server_legs_completed": 4, + "owned_server_exit0": 4, + "all_exact_live_prompt_ids": true, + "all_completion16_cached0": true, + "repeat_cells": 4, + "all_repeated_output_token_hashes_equal": true + }, + "provenance": { + "source_head_at_preparation": "07a7cb4cbc137259c1f53373e59e6d88d8ff8074", + "native_provenance": "Public applied artifact3d4 from attention-range receipt; git diff dd7263c26..07a7cb4 -- zig empty, later commits docs only", + "native_artifact": { + "public": { + "path": "$PACKET/artifacts/tensorfold-native", + "sha256": "3d4a3daeda697420bfbba6d81972896c2bf9e31efaf6f12321ee8487a299354b" + } + }, + "actual_native_version": { + "public": "tensorfold-native 1.1.6" + }, + "client_python_version": "3.13.15 (main, Aug 5 2026, 12:25:43) [Clang 21.0.0 (clang-2100.1.1.101)]", + "client_environment_requirements_sha256": "cdede98457af90bb37dc146204af09dc67059ae7829932d7bae76b9185740ee4", + "checkpoint_files_and_revisions": { + "LFM3-1.7B-MLX-4bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-4bit", + "revision": "d0ca73dcfa70ce18733754a492cdc1fb0123262d", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 2423, + "sha256": "7df2ea819f9a31b770192a0a9dc1b7953a7dd8c41b816cc3905bbec99ef332bd", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 328, + "sha256": "0095ba7158e938b1e2d3082d412f00d8be85e42cbbd7608b63866f7a081cf270", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 973083386, + "sha256": "1be5bf672013cd91aa62148f77c5eea8d2784a95845d5fe826e08c7b84823815", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 50700, + "sha256": "2c8b88e25f6a1055164dd051436b170872200047bce4b3ec5f2a58f6ef929a75", + "verified_against_pinned_revision": true + }, + "special_tokens_map.json": { + "bytes": 92, + "sha256": "7946f864cbfb62a1b041bda937244210a1a87f4a6cab3a48b791131b2cd57f88", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908634, + "sha256": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1859, + "sha256": "b77214b9f12be74b9e5e1dab35d7dace077a31be969314bf38866fd6c8163853", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-VL-2B-MLX-4bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "revision": "c823fade98e35a3d6942ad4d38b1ea15f675cd12", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 4419, + "sha256": "0efb7708e0771e3e104ac628954cdc71070dd436e0b7974615bd1b9eb836f9d9", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 323, + "sha256": "30b1115bcdc4e5671567a5b1255a2f3d2f5a912a79292051c6f1e9db81b5e8c4", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1590959088, + "sha256": "58268dfdf441ae4020d2a52fc97a814645e8a9624946e2d490403d4dcd71a255", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 61256, + "sha256": "38b756dbde167ab7c60756af7d5652414c31afdf4e77700b8d3c99395515d639", + "verified_against_pinned_revision": true + }, + "processor_config.json": { + "bytes": 1254, + "sha256": "f183f5a5e5b3a32e666fce05033e6cb7ca496d26e709a2cee611a13eb7d6b610", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908258, + "sha256": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1759, + "sha256": "d63d1ce2f332047e398764b4170352b96629a5194643105bfd6d70c90b750396", + "verified_against_pinned_revision": true + } + } + } + }, + "runner_sha256": "959eda8cc9ebba01f01d0850c3fb59f1bec26a6b9d288581ecc2f95617febe18", + "cold_client_sha256": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2", + "guard_sha256": "0caf67448d401704bd849744584248729decf5fd5b994f669a518cb61891df37" + }, + "protocol": { + "context": 70000, + "configured_parallel": 4, + "actual_simultaneous_requests": 1, + "prompt_cache_gib": 0, + "temperature": 0, + "thinking": false, + "completion_tokens": 16, + "families": [ + "text", + "vl" + ], + "bits": [ + 4 + ], + "cold_prompt_tokens": [ + 32768, + 65536 + ], + "fresh_server_order": [ + "text4", + "text4", + "VL4", + "VL4" + ], + "fresh_KV_each_request_verified_cached0": true, + "draft_field_in_saved_request": "absent; actual runtime draft setting retained per request", + "long_draft_false_comparison": false, + "long_concurrent_solo_comparison": false, + "memory_policy": "Existing native admission unchanged; no memory or wired limits raised. Process footprint measured per server across its32k and64k requests, not per-request GPU allocator peaks.", + "lock_events": [ + { + "event": "lock_wait", + "time_ns": 1793212362725465000, + "path": "/private/tmp/tensorfold-model.lock" + }, + { + "event": "lock_acquired", + "time_ns": 1793212362725689000, + "path": "/private/tmp/tensorfold-model.lock", + "pid": 19448, + "device": 16777231, + "inode": 46835039 + }, + { + "event": "complete", + "time_ns": 1793214795988082000, + "legs": 4, + "all_lengths_validated": true, + "performance_gate_claim": false + }, + { + "event": "lock_release", + "time_ns": 1793214795988270000, + "pid": 19448 + } + ], + "owned_load_ready_cleanup_events": [ + { + "event": "owned_model_load_begin", + "time_ns": 1793212364432858000, + "tag": "text-4-0-public", + "pid": 19484, + "command": [ + "$PACKET/artifacts/tensorfold-native", + "serve", + "$MODELS/LFM3-1.7B-MLX-4bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "event": "owned_model_ready", + "time_ns": 1793212366601124000, + "tag": "text-4-0-public", + "api_model": "lfm3-local", + "models": [ + { + "id": "lfm3-local", + "object": "model", + "created": 1793212366, + "owned_by": "tensorfold", + "context_length": 70000, + "max_model_len": 70000, + "reasoning": { + "default_enabled": false + } + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793212982412501000, + "tag": "text-4-0-public", + "pid": 19484, + "exit": 0, + "memory": { + "max_rss_bytes": 1231667200, + "peak_physical_footprint_bytes": 2818033392 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793212982639224000, + "tag": "text-4-1-public", + "pid": 29903, + "command": [ + "$PACKET/artifacts/tensorfold-native", + "serve", + "$MODELS/LFM3-1.7B-MLX-4bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "event": "owned_model_ready", + "time_ns": 1793212984797712000, + "tag": "text-4-1-public", + "api_model": "lfm3-local", + "models": [ + { + "id": "lfm3-local", + "object": "model", + "created": 1793212984, + "owned_by": "tensorfold", + "context_length": 70000, + "max_model_len": 70000, + "reasoning": { + "default_enabled": false + } + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793213588196102000, + "tag": "text-4-1-public", + "pid": 29903, + "exit": 0, + "memory": { + "max_rss_bytes": 1231994880, + "peak_physical_footprint_bytes": 2822227600 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793213588433067000, + "tag": "vl-4-0-public", + "pid": 40856, + "command": [ + "$PACKET/artifacts/tensorfold-native", + "serve", + "$MODELS/LFM3-VL-2B-MLX-4bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "event": "owned_model_ready", + "time_ns": 1793213591598905000, + "tag": "vl-4-0-public", + "api_model": "lfm3-local", + "models": [ + { + "id": "lfm3-local", + "object": "model", + "created": 1793213591, + "owned_by": "tensorfold", + "context_length": 70000, + "max_model_len": 70000, + "reasoning": { + "default_enabled": false + } + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793214192784840000, + "tag": "vl-4-0-public", + "pid": 40856, + "exit": 0, + "memory": { + "max_rss_bytes": 1850359808, + "peak_physical_footprint_bytes": 3478177952 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793214193015609000, + "tag": "vl-4-1-public", + "pid": 51402, + "command": [ + "$PACKET/artifacts/tensorfold-native", + "serve", + "$MODELS/LFM3-VL-2B-MLX-4bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "event": "owned_model_ready", + "time_ns": 1793214196162312000, + "tag": "vl-4-1-public", + "api_model": "lfm3-local", + "models": [ + { + "id": "lfm3-local", + "object": "model", + "created": 1793214196, + "owned_by": "tensorfold", + "context_length": 70000, + "max_model_len": 70000, + "reasoning": { + "default_enabled": false + } + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793214795872074000, + "tag": "vl-4-1-public", + "pid": 51402, + "exit": 0, + "memory": { + "max_rss_bytes": 1850261504, + "peak_physical_footprint_bytes": 3486206112 + } + } + ] + }, + "cells": [ + { + "family": "text", + "bits": 4, + "prompt_tokens": 32768, + "fresh_server_repetitions": 2, + "all_output_16_cached0": true, + "native_live_ids_equal_published": true, + "repeated_full_output_token_hash_equal": true, + "output_token_sha256": "e15e7490e640d938a0a846dfc9de1dd7a51e50ae97ca4fd42a213cbb633f66f5" + }, + { + "family": "text", + "bits": 4, + "prompt_tokens": 65536, + "fresh_server_repetitions": 2, + "all_output_16_cached0": true, + "native_live_ids_equal_published": true, + "repeated_full_output_token_hash_equal": true, + "output_token_sha256": "cf6c4062b60dc49e0ca503b7be2aaf3d247d73be037d269cf73e127a5e35aa32" + }, + { + "family": "vl", + "bits": 4, + "prompt_tokens": 32768, + "fresh_server_repetitions": 2, + "all_output_16_cached0": true, + "native_live_ids_equal_published": true, + "repeated_full_output_token_hash_equal": true, + "output_token_sha256": "f35dfbfd86653608bd3aa0fe6672229ecfe9ec17bf45ebcdb91dc7c7e894af4a" + }, + { + "family": "vl", + "bits": 4, + "prompt_tokens": 65536, + "fresh_server_repetitions": 2, + "all_output_16_cached0": true, + "native_live_ids_equal_published": true, + "repeated_full_output_token_hash_equal": true, + "output_token_sha256": "6dc17e16c1bdd265cbfecc43f466db1fb29c9240e7818d912ecc773b568b4856" + } + ], + "requests": [ + { + "family": "text", + "bits": 4, + "leg": 0, + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "native_live_ids_equal_published": true, + "published_ids_file": "docs/receipts/lfm3/cold-prompts/text-32768.ids.json", + "published_ids_json_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "actual_tokenize_response_sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450", + "published_body_file": "docs/receipts/lfm3/cold-prompts/text-32768.body.json", + "published_body_sha256": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "actual_sent_body_sha256": "8669dab27e5eb8320e6149804676a48eed58b6f7b2eee972fa92c563e180686d", + "output_token_sha256": "e15e7490e640d938a0a846dfc9de1dd7a51e50ae97ca4fd42a213cbb633f66f5", + "output_text_sha256": "382055acebd5e91b010ffa41b6497ebdb6d22c9bbdd7a2ab220c18df2e11f6b3", + "sampling": "greedy", + "runtime_drafts_setting": true, + "client_ttft_seconds": 139.58678591717035, + "client_elapsed_seconds": 139.8003973751329, + "engine_prefill_seconds": 139.575941459, + "raw_result_sha256": "8439a265a08546347e4935490734747684565af6f7fd96a1662c87ac3cdbc494" + }, + { + "family": "text", + "bits": 4, + "leg": 0, + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "native_live_ids_equal_published": true, + "published_ids_file": "docs/receipts/lfm3/cold-prompts/text-65536.ids.json", + "published_ids_json_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "actual_tokenize_response_sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082", + "published_body_file": "docs/receipts/lfm3/cold-prompts/text-65536.body.json", + "published_body_sha256": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "actual_sent_body_sha256": "5dbd5266aa3cd2635ba020f32f85bdb03ae71155c3114601b55cdd0007f5cfb6", + "output_token_sha256": "cf6c4062b60dc49e0ca503b7be2aaf3d247d73be037d269cf73e127a5e35aa32", + "output_text_sha256": "49b9640f88ffd97f99e1c6a354211b68d7b226874d3d8c1afbc261003f02cb39", + "sampling": "greedy", + "runtime_drafts_setting": true, + "client_ttft_seconds": 474.5117108747363, + "client_elapsed_seconds": 474.7762949159369, + "engine_prefill_seconds": 474.499127333, + "raw_result_sha256": "d09252928059098d7686f05a30c94ae0e3534d6e7905a4b9ec61d6a211fd14b1" + }, + { + "family": "text", + "bits": 4, + "leg": 1, + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "native_live_ids_equal_published": true, + "published_ids_file": "docs/receipts/lfm3/cold-prompts/text-32768.ids.json", + "published_ids_json_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "actual_tokenize_response_sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450", + "published_body_file": "docs/receipts/lfm3/cold-prompts/text-32768.body.json", + "published_body_sha256": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "actual_sent_body_sha256": "8669dab27e5eb8320e6149804676a48eed58b6f7b2eee972fa92c563e180686d", + "output_token_sha256": "e15e7490e640d938a0a846dfc9de1dd7a51e50ae97ca4fd42a213cbb633f66f5", + "output_text_sha256": "382055acebd5e91b010ffa41b6497ebdb6d22c9bbdd7a2ab220c18df2e11f6b3", + "sampling": "greedy", + "runtime_drafts_setting": true, + "client_ttft_seconds": 135.97505516605452, + "client_elapsed_seconds": 136.1665180833079, + "engine_prefill_seconds": 135.9630235, + "raw_result_sha256": "58310c03fccf8cea8286c2d2121aa19f7e7b9e062c07277ce56528d63ffc874d" + }, + { + "family": "text", + "bits": 4, + "leg": 1, + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "native_live_ids_equal_published": true, + "published_ids_file": "docs/receipts/lfm3/cold-prompts/text-65536.ids.json", + "published_ids_json_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "actual_tokenize_response_sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082", + "published_body_file": "docs/receipts/lfm3/cold-prompts/text-65536.body.json", + "published_body_sha256": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "actual_sent_body_sha256": "5dbd5266aa3cd2635ba020f32f85bdb03ae71155c3114601b55cdd0007f5cfb6", + "output_token_sha256": "cf6c4062b60dc49e0ca503b7be2aaf3d247d73be037d269cf73e127a5e35aa32", + "output_text_sha256": "49b9640f88ffd97f99e1c6a354211b68d7b226874d3d8c1afbc261003f02cb39", + "sampling": "greedy", + "runtime_drafts_setting": true, + "client_ttft_seconds": 465.85748220793903, + "client_elapsed_seconds": 466.141094957944, + "engine_prefill_seconds": 465.84401475, + "raw_result_sha256": "8877e6ff54c3923bcac03902f521a84967a75ceb72855b6d14318633ec56f671" + }, + { + "family": "vl", + "bits": 4, + "leg": 0, + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "native_live_ids_equal_published": true, + "published_ids_file": "docs/receipts/lfm3/cold-prompts/vl-32768.ids.json", + "published_ids_json_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "actual_tokenize_response_sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450", + "published_body_file": "docs/receipts/lfm3/cold-prompts/vl-32768.body.json", + "published_body_sha256": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "actual_sent_body_sha256": "8669dab27e5eb8320e6149804676a48eed58b6f7b2eee972fa92c563e180686d", + "output_token_sha256": "f35dfbfd86653608bd3aa0fe6672229ecfe9ec17bf45ebcdb91dc7c7e894af4a", + "output_text_sha256": "2acefce2656f36e21405907d799ceb7a85e7557924451ed2c622c79474fa0999", + "sampling": "greedy", + "runtime_drafts_setting": true, + "client_ttft_seconds": 145.4195410842076, + "client_elapsed_seconds": 145.65195962507278, + "engine_prefill_seconds": 145.406425, + "raw_result_sha256": "8328d2302a5fcc44b0c25b9794d6307a0e8493144165e8c46a99c1d6030f4627" + }, + { + "family": "vl", + "bits": 4, + "leg": 0, + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "native_live_ids_equal_published": true, + "published_ids_file": "docs/receipts/lfm3/cold-prompts/vl-65536.ids.json", + "published_ids_json_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "actual_tokenize_response_sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082", + "published_body_file": "docs/receipts/lfm3/cold-prompts/vl-65536.body.json", + "published_body_sha256": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "actual_sent_body_sha256": "5dbd5266aa3cd2635ba020f32f85bdb03ae71155c3114601b55cdd0007f5cfb6", + "output_token_sha256": "6dc17e16c1bdd265cbfecc43f466db1fb29c9240e7818d912ecc773b568b4856", + "output_text_sha256": "eb30855e72463651ef28f847b1464ebf1651787ab4add8a37e5bd90565054e97", + "sampling": "greedy", + "runtime_drafts_setting": true, + "client_ttft_seconds": 453.94360912498087, + "client_elapsed_seconds": 454.2238815417513, + "engine_prefill_seconds": 453.929644125, + "raw_result_sha256": "8cc0d5b657dc2494fb1851c41bfb31d86a5fd4795a34bd1ce3d8e62ea7513829" + }, + { + "family": "vl", + "bits": 4, + "leg": 1, + "prompt_tokens": 32768, + "completion_tokens": 16, + "cached_tokens": 0, + "native_live_ids_equal_published": true, + "published_ids_file": "docs/receipts/lfm3/cold-prompts/vl-32768.ids.json", + "published_ids_json_sha256": "66a7d4a0c49b2311500556e8e0f56cee9d0ae033f7765736b61d5a9c4e133c9f", + "actual_tokenize_response_sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450", + "published_body_file": "docs/receipts/lfm3/cold-prompts/vl-32768.body.json", + "published_body_sha256": "6e4c2ecd84a08e202dddce7f9b50bada5c006f9d529a74177f38dc11c08e0bb4", + "actual_sent_body_sha256": "8669dab27e5eb8320e6149804676a48eed58b6f7b2eee972fa92c563e180686d", + "output_token_sha256": "f35dfbfd86653608bd3aa0fe6672229ecfe9ec17bf45ebcdb91dc7c7e894af4a", + "output_text_sha256": "2acefce2656f36e21405907d799ceb7a85e7557924451ed2c622c79474fa0999", + "sampling": "greedy", + "runtime_drafts_setting": true, + "client_ttft_seconds": 132.31399954203516, + "client_elapsed_seconds": 132.5132726249285, + "engine_prefill_seconds": 132.301139041, + "raw_result_sha256": "24b7c72f2e09a206610a182551f143a5a04ef5bcc7f4106d626cc4846ee8b15e" + }, + { + "family": "vl", + "bits": 4, + "leg": 1, + "prompt_tokens": 65536, + "completion_tokens": 16, + "cached_tokens": 0, + "native_live_ids_equal_published": true, + "published_ids_file": "docs/receipts/lfm3/cold-prompts/vl-65536.ids.json", + "published_ids_json_sha256": "4d92c019e5bc47be53d2d4f9199d416b283fa5b72e746347add288619d48a5ed", + "actual_tokenize_response_sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082", + "published_body_file": "docs/receipts/lfm3/cold-prompts/vl-65536.body.json", + "published_body_sha256": "d6dea630f39ee2ddd13e8a88a90050d615d518e21f5a9219e1dc3d2297617b63", + "actual_sent_body_sha256": "5dbd5266aa3cd2635ba020f32f85bdb03ae71155c3114601b55cdd0007f5cfb6", + "output_token_sha256": "6dc17e16c1bdd265cbfecc43f466db1fb29c9240e7818d912ecc773b568b4856", + "output_text_sha256": "eb30855e72463651ef28f847b1464ebf1651787ab4add8a37e5bd90565054e97", + "sampling": "greedy", + "runtime_drafts_setting": true, + "client_ttft_seconds": 465.8405224168673, + "client_elapsed_seconds": 466.1292872498743, + "engine_prefill_seconds": 465.82749725, + "raw_result_sha256": "ed1470fa7ba46b436b662ed9eee87aa39c85634f606c8da363c9935112028e3a" + } + ], + "per_server_footprints": [ + { + "family": "text", + "bits": 4, + "leg": 0, + "max_rss_bytes": 1231667200, + "peak_physical_footprint_bytes": 2818033392 + }, + { + "family": "text", + "bits": 4, + "leg": 1, + "max_rss_bytes": 1231994880, + "peak_physical_footprint_bytes": 2822227600 + }, + { + "family": "vl", + "bits": 4, + "leg": 0, + "max_rss_bytes": 1850359808, + "peak_physical_footprint_bytes": 3478177952 + }, + { + "family": "vl", + "bits": 4, + "leg": 1, + "max_rss_bytes": 1850261504, + "peak_physical_footprint_bytes": 3486206112 + } + ], + "raw_file_manifest": { + "events.json": { + "bytes": 9853, + "sha256": "8e188ba422d7f63b551c8cc1501dfe377ca60b6e159eb71fe611dd09e51741b8" + }, + "manifest.json": { + "bytes": 12583, + "sha256": "acc389270cd5420597c41a50d02e402942ce3ff1c059dac988cce2b909da8984" + }, + "records.json": { + "bytes": 3876, + "sha256": "5f870598f57e044bb2a85d58e498069ea9d2f1250f7ea7dc96558c93db6e9ceb" + }, + "text-4-0-public.after.health": { + "bytes": 298, + "sha256": "9c1b816b0eb1a525cd199e124a13c970db89bfa5eb3feee98a434be7b6699b1e" + }, + "text-4-0-public.after.metrics": { + "bytes": 12200, + "sha256": "9a6ecda2d3f4bdf41b592f05939e48a3c615307b8265f190981a97bb13a1eaa3" + }, + "text-4-0-public.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-0-public.before.health": { + "bytes": 296, + "sha256": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881" + }, + "text-4-0-public.before.metrics": { + "bytes": 12128, + "sha256": "9cc5e25cd64db521fbd3ce28eeff18076a2f4a0f879a19c235cb03562c224102" + }, + "text-4-0-public.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-0-public.cold-32768.actual-tokenize.json": { + "bytes": 229416, + "sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450" + }, + "text-4-0-public.cold-32768.client.log": { + "bytes": 350, + "sha256": "e41c00cb479e1c2fdc415ee729a6348ab4a7441e2f5cbe1334dab6da33f5a322" + }, + "text-4-0-public.cold-32768.json": { + "bytes": 204611, + "sha256": "8439a265a08546347e4935490734747684565af6f7fd96a1662c87ac3cdbc494" + }, + "text-4-0-public.cold-65536.actual-tokenize.json": { + "bytes": 458792, + "sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082" + }, + "text-4-0-public.cold-65536.client.log": { + "bytes": 349, + "sha256": "65446d1f66f538a305f97b7a18d7c19cb01cf69291c2addc6f64146974149bef" + }, + "text-4-0-public.cold-65536.json": { + "bytes": 401225, + "sha256": "d09252928059098d7686f05a30c94ae0e3534d6e7905a4b9ec61d6a211fd14b1" + }, + "text-4-0-public.server.log": { + "bytes": 2863, + "sha256": "84b0cf8c09526df94ad38acfab7059ce0f86d27539e8cc1ea5ec439bf04297d9" + }, + "text-4-1-public.after.health": { + "bytes": 298, + "sha256": "7e994d2238d049d9c0896261bf62aa264236336a7779efe0b06b110bc6f290db" + }, + "text-4-1-public.after.metrics": { + "bytes": 12195, + "sha256": "68cef663e4ed0b733c6bab028d6707eca56e57b6d00f9a8963d0791cda38341e" + }, + "text-4-1-public.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-1-public.before.health": { + "bytes": 296, + "sha256": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881" + }, + "text-4-1-public.before.metrics": { + "bytes": 12128, + "sha256": "07dab59d95ce9e3ff675912eda10439f0d5312bf7ff493c2f3b954adb2001ee3" + }, + "text-4-1-public.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-1-public.cold-32768.actual-tokenize.json": { + "bytes": 229416, + "sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450" + }, + "text-4-1-public.cold-32768.client.log": { + "bytes": 350, + "sha256": "9fc3f47eea5f7041ecaa512ffc98f4fc4e113da1a9ecdc235b6fc6434e8cb5b4" + }, + "text-4-1-public.cold-32768.json": { + "bytes": 204608, + "sha256": "58310c03fccf8cea8286c2d2121aa19f7e7b9e062c07277ce56528d63ffc874d" + }, + "text-4-1-public.cold-65536.actual-tokenize.json": { + "bytes": 458792, + "sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082" + }, + "text-4-1-public.cold-65536.client.log": { + "bytes": 349, + "sha256": "2245a3be6b6c94c18056ea94f674285dae8193d649ccbfd06d4bcba3ff838b7b" + }, + "text-4-1-public.cold-65536.json": { + "bytes": 401223, + "sha256": "8877e6ff54c3923bcac03902f521a84967a75ceb72855b6d14318633ec56f671" + }, + "text-4-1-public.server.log": { + "bytes": 2863, + "sha256": "2b70209f2525aca2348c9bba24e97491301f35c1e77b6854791514b3ad4f849b" + }, + "vl-4-0-public.after.health": { + "bytes": 298, + "sha256": "7f5940a84963cdf6dc32d38c332269b420e17dd518e22c258a06f4ca9e637bc4" + }, + "vl-4-0-public.after.metrics": { + "bytes": 12200, + "sha256": "5acd0131f7f835870f392563a1fd7d8085bbd002e0ebddb2a001324cf51f8fad" + }, + "vl-4-0-public.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-0-public.before.health": { + "bytes": 296, + "sha256": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881" + }, + "vl-4-0-public.before.metrics": { + "bytes": 12128, + "sha256": "07419319a246cf380cab00ccb870a7218a0db42c2a37d6062128ad1b591e1df7" + }, + "vl-4-0-public.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-0-public.cold-32768.actual-tokenize.json": { + "bytes": 229416, + "sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450" + }, + "vl-4-0-public.cold-32768.client.log": { + "bytes": 350, + "sha256": "5819197d4bfa1c0311797a1ada0388dc884658cb8ce00f4192e953abe180a7db" + }, + "vl-4-0-public.cold-32768.json": { + "bytes": 204609, + "sha256": "8328d2302a5fcc44b0c25b9794d6307a0e8493144165e8c46a99c1d6030f4627" + }, + "vl-4-0-public.cold-65536.actual-tokenize.json": { + "bytes": 458792, + "sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082" + }, + "vl-4-0-public.cold-65536.client.log": { + "bytes": 350, + "sha256": "45d3f632481267081e3c53afa0117b8ba63303325432267912b62620973d0074" + }, + "vl-4-0-public.cold-65536.json": { + "bytes": 401229, + "sha256": "8cc0d5b657dc2494fb1851c41bfb31d86a5fd4795a34bd1ce3d8e62ea7513829" + }, + "vl-4-0-public.server.log": { + "bytes": 2864, + "sha256": "5912c925ec3d51d41938cd6fd8b8a44d3565dd7b331bb38935284eaf85f20e64" + }, + "vl-4-1-public.after.health": { + "bytes": 298, + "sha256": "7e994d2238d049d9c0896261bf62aa264236336a7779efe0b06b110bc6f290db" + }, + "vl-4-1-public.after.metrics": { + "bytes": 12198, + "sha256": "d3a1469bb6833f555a20db304f3f6e289e5d8c30160e579eb075a7b5849d1057" + }, + "vl-4-1-public.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-1-public.before.health": { + "bytes": 296, + "sha256": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881" + }, + "vl-4-1-public.before.metrics": { + "bytes": 12128, + "sha256": "e5434eddfcbbf9f7220903f4888446b01bf95fc4676b2e926e205cb73daf5400" + }, + "vl-4-1-public.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-1-public.cold-32768.actual-tokenize.json": { + "bytes": 229416, + "sha256": "d4bfe4925faa3dc8627a5d23bafd579e19ec435abe5fb7db0df6641b8a155450" + }, + "vl-4-1-public.cold-32768.client.log": { + "bytes": 350, + "sha256": "420ac68f6e0103dab109ab73d7c8b841a9bdf372684bf91972d8aa83e361ce10" + }, + "vl-4-1-public.cold-32768.json": { + "bytes": 204612, + "sha256": "24b7c72f2e09a206610a182551f143a5a04ef5bcc7f4106d626cc4846ee8b15e" + }, + "vl-4-1-public.cold-65536.actual-tokenize.json": { + "bytes": 458792, + "sha256": "c1f5722c15f4f4ba0f97117e14b8ccda153d4bce407b23ed86f28cb91946a082" + }, + "vl-4-1-public.cold-65536.client.log": { + "bytes": 349, + "sha256": "ed726e4880a2b2856ec36560fc411b56ea92a9aba4b11548ff819901f657c26c" + }, + "vl-4-1-public.cold-65536.json": { + "bytes": 401228, + "sha256": "ed1470fa7ba46b436b662ed9eee87aa39c85634f606c8da363c9935112028e3a" + }, + "vl-4-1-public.server.log": { + "bytes": 2864, + "sha256": "ef7bdbe921dcd04489196d7e632ec49b0b833aee8e36a80ad5aca39f835159c1" + } + } + } + ], + "limitations": [ + "VL checkpoint requests contain text only. No long-context image acceptance or image-throughput claim.", + "Repeated replies establish deterministic fresh execution on these exact requests, not an independent FP64 teacher-forced model gate.", + "No same-machine standard or before/after timing comparison here; timings are native context diagnostics only.", + "This local48GiB run covers4bit only. M5 or larger-memory results do not qualify unmeasured6/8bit memory on this host.", + "No128k or maximum native-window/context acceptance from these32k/64k checks.", + "Configured parallel4 reserves the serving setting, but these long requests run sequentially; no long-context four-stream concurrency claim." + ], + "raw_retention": "Full exact native/tokenize responses, public input bodies/arrays, all output/runtime chunks, process logs and model/environment manifests retained privately and hashed. Private paths and request IDs are omitted from publication." +} diff --git a/docs/receipts/lfm3/decode-concurrency.json b/docs/receipts/lfm3/decode-concurrency.json new file mode 100644 index 0000000000..78f991ac5b --- /dev/null +++ b/docs/receipts/lfm3/decode-concurrency.json @@ -0,0 +1,168 @@ +{ + "native_sha256": "5c8a04d4af599879797f4f5bed8e00ab7d9d70732e698bd2a1ec2a01ec762e09", + "variant": "strict row-axis only; original attention", + "families": { + "text": { + "decode": [ + { + "prompt": "fibonacci-raw-chat", + "decode_tps_median": 317.6004846170962, + "ttft_s_median": 0.011276375036686659 + }, + { + "prompt": "gpu-chat-no-think-chat", + "decode_tps_median": 316.8637201461089, + "ttft_s_median": 0.012718458194285631 + } + ], + "concurrency": [ + { + "prompt": "code-chat", + "streams": 1, + "aggregate_tps": 316.9, + "failed": 0, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + } + }, + { + "prompt": "code-chat", + "streams": 4, + "aggregate_tps": 538.4, + "failed": 0, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + } + }, + { + "prompt": "chat-chat", + "streams": 1, + "aggregate_tps": 317.3, + "failed": 0, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + } + }, + { + "prompt": "chat-chat", + "streams": 4, + "aggregate_tps": 530.6, + "failed": 0, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + } + } + ], + "peak_physical_footprint_bytes": 1476495016 + }, + "vl": { + "decode": [ + { + "prompt": "fibonacci-raw-chat", + "decode_tps_median": 302.95882338372087, + "ttft_s_median": 0.011537457816302776 + }, + { + "prompt": "gpu-chat-no-think-chat", + "decode_tps_median": 297.51584456535494, + "ttft_s_median": 0.012835666071623564 + } + ], + "concurrency": [ + { + "prompt": "code-chat", + "streams": 1, + "aggregate_tps": 303.0, + "failed": 0, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + } + }, + { + "prompt": "code-chat", + "streams": 4, + "aggregate_tps": 536.4, + "failed": 0, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + } + }, + { + "prompt": "chat-chat", + "streams": 1, + "aggregate_tps": 296.6, + "failed": 0, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + } + }, + { + "prompt": "chat-chat", + "streams": 4, + "aggregate_tps": 521.8, + "failed": 0, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + } + } + ], + "peak_physical_footprint_bytes": 2136000672 + } + } +} diff --git a/docs/receipts/lfm3/earliest-stage-rounding.json b/docs/receipts/lfm3/earliest-stage-rounding.json new file mode 100644 index 0000000000..3198cf5c0c --- /dev/null +++ b/docs/receipts/lfm3/earliest-stage-rounding.json @@ -0,0 +1,400 @@ +{ + "scope": "First actual divergence on identical input rows, with production reduction rounding and unchanged existing bounds; not whole-model acceptance", + "cases": [ + { + "case": "realcat8", + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "native_diagnostic_sha256": "a6f6d5c4dfa719d9a2e90526a210bae07641da158c39b7772c200d0a1f17b468", + "embedding_equal": 184320, + "first_norm_equal": 184320, + "first_norm_values": 184320, + "projection": { + "values": 552960, + "replay_bf16_equal": 552960, + "replay_vs_fp64_raw_bound_exceedances": 0, + "native_vs_fp64_store_bound_exceedances": 0, + "existing_contract_depth": 36, + "worst_raw_bound_fraction": 0.004461910283213048, + "first_boundary_crossing": { + "row": 0, + "column": 5962, + "fp64": -6.937049874977674e-05, + "replayed_fp32": -6.938725709915161e-05, + "ideal_bf16": -6.914138793945312e-05, + "native_bf16": -6.961822509765625e-05, + "raw_error": 1.6758349374867976e-08, + "existing_raw_bound": 5.486111173345787e-05, + "factorized_magnitude": 25.567076293120408 + }, + "actual_raw_fp32_replay_equal": 552960, + "actual_raw_fp32_values": 552960, + "actual_raw_fp32_max_error_vs_replay": 0.0, + "actual_raw_fp32_sha256": "b0ae65908e77508efa4af926ab951461db3f9e7df00d0f9bd8bd20941a60c4b1", + "helper_source_sha256": "86963d6266a982f4cac7c85265048703841a9545c19f2d2c5271eb8e02a0879b" + }, + "same_native_input_conv": { + "values": 184320, + "fp32_replay_equal": 184320, + "max_error": 0.0 + }, + "guided_first_two_layers": { + "arithmetic": "Independent FP32 affine, RMS norm and convolution reduction replay; other operations retain FP64", + "stages": [ + { + "stage": "layer_00_act", + "values": 691200, + "equal_values": 691200 + }, + { + "stage": "layer_00_conv_gated", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_00_conv_in", + "values": 552960, + "equal_values": 552960 + }, + { + "stage": "layer_00_down", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_00_ffn_norm", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_00_gate", + "values": 691200, + "equal_values": 691200 + }, + { + "stage": "layer_00_mixer", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_00_operator_norm", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_00_output", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_00_post_operator", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_00_up", + "values": 691200, + "equal_values": 691200 + }, + { + "stage": "layer_01_act", + "values": 691200, + "equal_values": 691200 + }, + { + "stage": "layer_01_conv_gated", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_01_conv_in", + "values": 552960, + "equal_values": 552960 + }, + { + "stage": "layer_01_down", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_01_ffn_norm", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_01_gate", + "values": 691200, + "equal_values": 691200 + }, + { + "stage": "layer_01_mixer", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_01_operator_norm", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_01_output", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_01_post_operator", + "values": 184320, + "equal_values": 184320 + }, + { + "stage": "layer_01_up", + "values": 691200, + "equal_values": 691200 + } + ], + "values": 7833600, + "equal_values": 7833600 + } + }, + { + "case": "reasoning64-vl6", + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-6bit", + "native_diagnostic_sha256": "a6f6d5c4dfa719d9a2e90526a210bae07641da158c39b7772c200d0a1f17b468", + "embedding_equal": 131072, + "first_norm_equal": 131072, + "first_norm_values": 131072, + "projection": { + "values": 393216, + "replay_bf16_equal": 393216, + "replay_vs_fp64_raw_bound_exceedances": 0, + "native_vs_fp64_store_bound_exceedances": 0, + "existing_contract_depth": 36, + "worst_raw_bound_fraction": 0.0038237892749845067, + "first_boundary_crossing": { + "row": 0, + "column": 246, + "fp64": 0.0001032436775858514, + "replayed_fp32": 0.00010323524475097656, + "ideal_bf16": 0.00010347366333007812, + "native_bf16": 0.000102996826171875, + "raw_error": 8.43283487483859e-09, + "existing_raw_bound": 6.220623788976059e-05, + "factorized_magnitude": 28.990145838870376 + }, + "actual_raw_fp32_replay_equal": 393216, + "actual_raw_fp32_values": 393216, + "actual_raw_fp32_max_error_vs_replay": 0.0, + "actual_raw_fp32_sha256": "9e7b26fa0a4e1b6e77dc7a76c3437a2e4444abcade4fd905a2632acb65300da4", + "helper_source_sha256": "86963d6266a982f4cac7c85265048703841a9545c19f2d2c5271eb8e02a0879b" + }, + "same_native_input_conv": { + "values": 131072, + "fp32_replay_equal": 131072, + "max_error": 0.0 + }, + "guided_first_two_layers": { + "arithmetic": "Independent FP32 affine, RMS norm and convolution reduction replay; other operations retain FP64", + "stages": [ + { + "stage": "layer_00_act", + "values": 491520, + "equal_values": 491520 + }, + { + "stage": "layer_00_conv_gated", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_00_conv_in", + "values": 393216, + "equal_values": 393216 + }, + { + "stage": "layer_00_down", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_00_ffn_norm", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_00_gate", + "values": 491520, + "equal_values": 491520 + }, + { + "stage": "layer_00_mixer", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_00_operator_norm", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_00_output", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_00_post_operator", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_00_up", + "values": 491520, + "equal_values": 491520 + }, + { + "stage": "layer_01_act", + "values": 491520, + "equal_values": 491520 + }, + { + "stage": "layer_01_conv_gated", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_01_conv_in", + "values": 393216, + "equal_values": 393216 + }, + { + "stage": "layer_01_down", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_01_ffn_norm", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_01_gate", + "values": 491520, + "equal_values": 491520 + }, + { + "stage": "layer_01_mixer", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_01_operator_norm", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_01_output", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_01_post_operator", + "values": 131072, + "equal_values": 131072 + }, + { + "stage": "layer_01_up", + "values": 491520, + "equal_values": 491520 + } + ], + "values": 5570560, + "equal_values": 5570560 + } + } + ], + "raw_capture": "Diagnostic Metal tap calls unchanged production tf_rp_rows_at<4,2,BITS,false> before BF16 storage; every rounded captured output matches native.", + "query_underflow_observation": { + "case": "realcat8", + "layer": 2, + "values": 184320, + "differing_values": 470, + "columns": [ + 56, + 59, + 108 + ], + "all_differing_native_values_zero": true, + "maximum_absolute_error": 1.1663108012064884e-38, + "examples": [ + { + "row": 0, + "head": 1, + "column": 59, + "reference": -7.071333204165323e-39, + "native": 0.0 + }, + { + "row": 0, + "head": 3, + "column": 59, + "reference": -7.989688165745235e-39, + "native": 0.0 + }, + { + "row": 0, + "head": 14, + "column": 56, + "reference": -2.938735877055719e-39, + "native": 0.0 + }, + { + "row": 0, + "head": 14, + "column": 59, + "reference": 1.0101904577379033e-39, + "native": 0.0 + }, + { + "row": 0, + "head": 14, + "column": 108, + "reference": 1.0285575569695016e-38, + "native": 0.0 + }, + { + "row": 1, + "head": 1, + "column": 56, + "reference": -3.03057137321371e-39, + "native": -0.0 + }, + { + "row": 1, + "head": 1, + "column": 59, + "reference": -1.285696946211877e-39, + "native": 0.0 + }, + { + "row": 1, + "head": 1, + "column": 108, + "reference": 5.969307250269429e-39, + "native": 0.0 + }, + { + "row": 1, + "head": 3, + "column": 59, + "reference": -9.09171411964113e-39, + "native": 0.0 + }, + { + "row": 1, + "head": 14, + "column": 56, + "reference": 4.408103815583578e-39, + "native": 0.0 + } + ], + "scope": "Observed native zeros versus preserved BF16 subnormals in the CPU reference; not an end-to-end acceptance waiver" + } +} diff --git a/docs/receipts/lfm3/fidelity-investigation.json b/docs/receipts/lfm3/fidelity-investigation.json new file mode 100644 index 0000000000..c4e7a8341e --- /dev/null +++ b/docs/receipts/lfm3/fidelity-investigation.json @@ -0,0 +1,184 @@ +{ + "checkpoint_policy_evidence": { + "mlx_version": "0.32.4", + "official_source_commit": "36969c094354086e15d2a327a9c50feb20b9cf11", + "source_tag": "v0.32.4", + "device": { + "device_name": "Apple M4 Pro", + "max_recommended_working_set_size": 40200896512, + "memory_size": 51539607552, + "architecture": "applegpu_g16s", + "max_buffer_length": 30150672384, + "resource_limit": 499000 + }, + "projection": "language_model.model.layers.0.conv.in_proj", + "bits": 8, + "group_size": 64, + "K": 2048, + "N": 6144, + "input": "same first BF16 operator_norm row repeated M times", + "results": [ + { + "rows": 1, + "dtype": "mlx.core.bfloat16", + "same_as_exact_dequant_FP64_rounded": 6144, + "same_as_BF16_weights_FP64_rounded": 3803, + "outputs": 6144, + "rms_vs_exact_FP64": 0.0003453614152093189 + }, + { + "rows": 2, + "dtype": "mlx.core.bfloat16", + "same_as_exact_dequant_FP64_rounded": 6144, + "same_as_BF16_weights_FP64_rounded": 3803, + "outputs": 6144, + "rms_vs_exact_FP64": 0.0003453614152093189 + }, + { + "rows": 4, + "dtype": "mlx.core.bfloat16", + "same_as_exact_dequant_FP64_rounded": 6144, + "same_as_BF16_weights_FP64_rounded": 3803, + "outputs": 6144, + "rms_vs_exact_FP64": 0.0003453614152093189 + }, + { + "rows": 8, + "dtype": "mlx.core.bfloat16", + "same_as_exact_dequant_FP64_rounded": 6144, + "same_as_BF16_weights_FP64_rounded": 3803, + "outputs": 6144, + "rms_vs_exact_FP64": 0.0003453614152093189 + }, + { + "rows": 12, + "dtype": "mlx.core.bfloat16", + "same_as_exact_dequant_FP64_rounded": 6144, + "same_as_BF16_weights_FP64_rounded": 3803, + "outputs": 6144, + "rms_vs_exact_FP64": 0.0003453614152093189 + }, + { + "rows": 13, + "dtype": "mlx.core.bfloat16", + "same_as_exact_dequant_FP64_rounded": 3803, + "same_as_BF16_weights_FP64_rounded": 6143, + "outputs": 6144, + "rms_vs_exact_FP64": 0.0004261891067811766 + }, + { + "rows": 16, + "dtype": "mlx.core.bfloat16", + "same_as_exact_dequant_FP64_rounded": 3803, + "same_as_BF16_weights_FP64_rounded": 6143, + "outputs": 6144, + "rms_vs_exact_FP64": 0.0004261891067811766 + }, + { + "rows": 32, + "dtype": "mlx.core.bfloat16", + "same_as_exact_dequant_FP64_rounded": 3803, + "same_as_BF16_weights_FP64_rounded": 6143, + "outputs": 6144, + "rms_vs_exact_FP64": 0.0004261891067811766 + }, + { + "rows": 64, + "dtype": "mlx.core.bfloat16", + "same_as_exact_dequant_FP64_rounded": 3803, + "same_as_BF16_weights_FP64_rounded": 6144, + "outputs": 6144, + "rms_vs_exact_FP64": 0.0004261898084132233 + } + ], + "M1_M64_exact_equal": 3803, + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "checkpoint_revision": "83336d5be3874a2bdd3297bde77b2114dbd3a816", + "fixture_sha256": "9c3636ee22fb5d1a9d4d6e3cc0b8e47cce20cb4288bb16f2970de8d55ee27708", + "source_links": [ + "https://github.com/ml-explore/mlx/blob/36969c094354086e15d2a327a9c50feb20b9cf11/mlx/backend/metal/quantized.cpp#L89", + "https://github.com/ml-explore/mlx/blob/36969c094354086e15d2a327a9c50feb20b9cf11/mlx/backend/metal/quantized.cpp#L1900", + "https://github.com/ml-explore/mlx/blob/36969c094354086e15d2a327a9c50feb20b9cf11/mlx/backend/metal/kernels/quantized.h#L555", + "https://github.com/ml-explore/mlx/blob/36969c094354086e15d2a327a9c50feb20b9cf11/mlx/backend/metal/kernels/quantized.h#L674", + "https://github.com/ml-explore/mlx/blob/36969c094354086e15d2a327a9c50feb20b9cf11/mlx/backend/metal/kernels/quantized.h#L851" + ], + "interpretation": "Official MLX uses FP32 affine factorization for these vector widths and BF16 reconstructed weights for these matrix widths. Native uses one affine reconstruction policy across prompt and decode. This evidence alone is not a model-fidelity acceptance policy." + }, + "projection_error_audit": { + "cells": 66, + "outputs": 60954624, + "raw_tap_matches_all_native": true, + "one_row_tap_matches_all_native_first_rows": true, + "raw_FP32_bound_exceedances": 0, + "BF16_bound_exceedances": 0, + "max_raw_FP32_bound_fraction": 0.01465208540032716, + "max_BF16_bound_fraction": 0.9807553251550621, + "max_raw_abs_error": 2.555781975388527e-06, + "max_worst_column_raw_rms": 7.7385387436973e-07, + "model_acceptance_threshold": null + }, + "first_actual_miss_primitive": { + "source_sha256": "86963d6266a982f4cac7c85265048703841a9545c19f2d2c5271eb8e02a0879b", + "formula": "shared tf_rp_rows_at<4,2,BITS,false>; FP32 accumulator captured before BF16 store", + "tap_rounds_to_native_bytes": true, + "row": 1, + "column": 4373, + "expected_FP64": -6.1827595345675945e-06, + "actual_FP32": -6.18956983089447e-06, + "error": -6.810296326875687e-09, + "ideal_BF16": -6.16908073425293e-06, + "native_BF16": -6.198883056640625e-06, + "existing_contract_depth": 36, + "factorized_term_magnitude": 25.70957951276796, + "existing_contract_FP32_work_bound": 5.5166891125901165e-05, + "fraction_of_work_bound": 0.00012344897796276604 + }, + "causal_history_isolation": { + "row": 17, + "native_final_norm_FP64_head_maximum_set": [ + 296 + ], + "native_maximum_set": [ + 296 + ], + "whole_FP64_model_maximum_set": [ + 405 + ], + "native_input_FP64_head_matches_native": 127999, + "head_values": 128000, + "same_input_conv": [ + { + "layer": 0, + "same_native_inputs_correctly_rounded_values": 526329, + "values": 526336, + "row17_native_input_output_differences": 0, + "row17_max_abs_error": 0.0, + "max_abs_error": 4.76837158203125e-07 + }, + { + "layer": 1, + "same_native_inputs_correctly_rounded_values": 526328, + "values": 526336, + "row17_native_input_output_differences": 0, + "row17_max_abs_error": 0.0, + "max_abs_error": 1.9073486328125e-06 + } + ] + }, + "whole_model_direct_FP64_comparison": { + "cells": 17, + "rows": 18, + "native_in_oracle_maximum": 9, + "mlx_in_oracle_maximum": 16, + "both_in_maximum": 8, + "neither_in_maximum": 1, + "accepted_threshold": null + }, + "interpretation": [ + "The audited shared projection implementation obeys its existing FP32 and BF16 store error budgets on the measured input classes.", + "The real mismatch traces to a measured FP32 accumulation error crossing a BF16 boundary and propagating through causal history.", + "Official MLX uses different effective reconstruction arithmetic for vector and matrix paths on the measured projection.", + "No new end-to-end model-fidelity tolerance or tie policy is defined by this evidence.", + "Model-fidelity acceptance remains unresolved; a prospective faster projection must be compared independently before adoption." + ] +} diff --git a/docs/receipts/lfm3/fidelity-prompts/code-257.ids.json b/docs/receipts/lfm3/fidelity-prompts/code-257.ids.json new file mode 100644 index 0000000000..b5c9d5c2cf --- /dev/null +++ b/docs/receipts/lfm3/fidelity-prompts/code-257.ids.json @@ -0,0 +1 @@ +[22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129] diff --git a/docs/receipts/lfm3/fidelity-prompts/code-513.ids.json b/docs/receipts/lfm3/fidelity-prompts/code-513.ids.json new file mode 100644 index 0000000000..8c2cb799d6 --- /dev/null +++ b/docs/receipts/lfm3/fidelity-prompts/code-513.ids.json @@ -0,0 +1 @@ +[22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318] diff --git a/docs/receipts/lfm3/fidelity-prompts/code-64.ids.json b/docs/receipts/lfm3/fidelity-prompts/code-64.ids.json new file mode 100644 index 0000000000..0fdc90d682 --- /dev/null +++ b/docs/receipts/lfm3/fidelity-prompts/code-64.ids.json @@ -0,0 +1 @@ +[22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162, 33237, 116766, 92377, 20, 1583, 4673, 360, 1224, 550, 34289, 360, 723, 550, 495, 550, 229, 24, 207, 22497, 267, 21516, 1446, 395, 16033, 273, 1046, 38076, 15493, 2129, 8864, 3520, 5635, 22, 42847, 851, 893, 309, 2993, 16318, 63908, 3162] diff --git a/docs/receipts/lfm3/fidelity-prompts/manifest.json b/docs/receipts/lfm3/fidelity-prompts/manifest.json new file mode 100644 index 0000000000..2e6e92b679 --- /dev/null +++ b/docs/receipts/lfm3/fidelity-prompts/manifest.json @@ -0,0 +1,119 @@ +[ + { + "name": "prose-64.npy", + "source_text": "Explain why the sky is blue, then describe how the color changes at sunset. Give a clear explanation using ordinary words. ", + "repeat_count": 20, + "special_tokens": false, + "tokenizer": "LiquidAI/LFM3-1.7B-MLX-4bit/tokenizer.json", + "nominal_token_count": 64, + "actual_token_count": 64, + "dtype": " Affine.one; direct call bypassed the separately parallel Affine.reference", + "change": "Independent flat logical-output jobs only; unchanged coordinates, q.one, clipped GLU and magnitude order", + "serial_reference_retained": true, + "entries": 78, + "bits": [ + 2, + 3, + 4, + 5, + 6, + 8 + ], + "kinds": [ + "narrow", + "wide", + "down", + "glu" + ], + "modes": [ + "oracle", + "replacement", + "control", + "up_oracle where defined" + ], + "row_counts": [ + 3, + 4 + ], + "groups": [ + 32, + 64 + ], + "fills": [ + "random", + "stress", + "nonfinite" + ], + "mutations": [ + "input", + "codes", + "scales", + "up input/codes", + "order duplicate", + "consistent expert boundaries" + ], + "fixture_states": 936, + "raw_want_magnitude_pair_checks": 9360, + "worker_requests": [ + 1, + 3, + 8 + ], + "spawn_modes": [ + "normal", + "partial_failure", + "all_failure" + ], + "fixture_allocator_unused": true, + "verification": { + "existing_glm_host_tests_passed": 19, + "single_threaded_fixture_tests_passed": 1, + "final_mutation_filtered_host_tests_passed": 1, + "replacement_binary_sha256": "fb93eb182eb46d13ebbc507cc4448d75d9463f5d7f7894d7b8d3b5d3084df991" + }, + "source_sha256": { + "zig/tests/contract/glm_rows_fixture.zig": "5bc793d71cd9d0113729a9f389f1ced0e903dc2e7745e785d670c9350dabd207", + "zig/glm_rows_contract_host.zig": "f81ab35aa1fabbe777a297ab163a0ed512a905adf4ebcf037fefadfdf99de4b4", + "zig/src/core/contract/reference_threads.zig": "395537771d6086fdd9298a68f4cdc2c33a3db59ed146cf0fd419d70f6b8233bc" + }, + "performance": "No timing or whole-suite speedup claimed for this extension; next immutable full run supplies actual completion evidence" + } +} diff --git a/docs/receipts/lfm3/greedy-dual.json b/docs/receipts/lfm3/greedy-dual.json new file mode 100644 index 0000000000..ec713d7156 --- /dev/null +++ b/docs/receipts/lfm3/greedy-dual.json @@ -0,0 +1,24009 @@ +{ + "scope": "Byte-preserving paired FFN gate/up projection plus exact CPU unmasked greedy selection. No tensor-unit attention qualification or agreed performance gate pass is asserted.", + "candidate_sha256": "b804c6ed1859710a7766f66480d569f163e3287b2eb1f8fc6a7ea29434e0562c", + "baseline_sha256": "d1656249bdb581b147faacbd3b3c557c00b6c45b710eeddce517a67178b6ba2b", + "proof_forward_sha256": "a05dfd241f67e05ab5ba8a0ea978c094b889818a463edab80f29848ceb25d4e3", + "source_hash_scope": "These hashes identify the GPU operator and greedy function; they are not a whole-tree build manifest.", + "gpu_operator_source_sha256": "05a21d65ec7563d9fadecf49bd88269b17d351591f674a5359c5aacccbfc6601", + "greedy_function_sha256": "2c445419e11cd674dbf88289654a1ce87d04c49e9081d726c515bbb8234cad9e", + "contract": { + "hardware": "M4 Pro, 48 GB, macOS 26.5.1", + "exit_status": 0, + "summary": "SUMMARY 58 kernels: 58 pass gates 1-4, 0 fail, 0 known findings; 16 controls: 16 red as planted, 0 missed; 0 limit problems; gates 62629 ms, limits 127 ms" + }, + "exactness": { + "hardware": "M4 Pro, 48 GB, macOS 26.5.1", + "checkpoint": "LFM3-1.7B-MLX-4bit", + "exit_status": 0, + "summary": "LFM3_EXACT mode=default passed=68 failed=0 skipped=0 prompt=5 windows=60 shared=2 snapshots=1 long=0 gpu_seconds=141.060" + }, + "cpu_greedy": { + "tests": 3, + "exit_status": 0, + "build": "Zig 0.17.0, -Ofast", + "scope": "All finite BF16 encodings and adjacent pairs in both orders, every nonfinite encoding, signed zero, vector/remainder boundaries, randomized full vocabularies. Masked and positive-temperature sampling retain the original path." + }, + "actual_projection_pairs": { + "cases": 132, + "distinct_weight_cases": 120, + "identical_bytes": 247628160, + "cells": [ + { + "name": "1.7B-4-conv_in", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.0.conv.in_proj", + "module_b": "language_model.model.layers.1.conv.in_proj", + "distinct_weights": true, + "rows": 1, + "n": 6144, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 24576 + }, + { + "name": "1.7B-4-conv_in", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.0.conv.in_proj", + "module_b": "language_model.model.layers.1.conv.in_proj", + "distinct_weights": true, + "rows": 64, + "n": 6144, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 1572864 + }, + { + "name": "1.7B-4-conv_out", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.0.conv.out_proj", + "module_b": "language_model.model.layers.1.conv.out_proj", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "1.7B-4-conv_out", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.0.conv.out_proj", + "module_b": "language_model.model.layers.1.conv.out_proj", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "1.7B-4-ffn_gate", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.0.feed_forward.w1", + "module_b": "language_model.model.layers.0.feed_forward.w3", + "distinct_weights": true, + "rows": 1, + "n": 7680, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 30720 + }, + { + "name": "1.7B-4-ffn_gate", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.0.feed_forward.w1", + "module_b": "language_model.model.layers.0.feed_forward.w3", + "distinct_weights": true, + "rows": 64, + "n": 7680, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 1966080 + }, + { + "name": "1.7B-4-ffn_up", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.0.feed_forward.w3", + "module_b": "language_model.model.layers.0.feed_forward.w1", + "distinct_weights": true, + "rows": 1, + "n": 7680, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 30720 + }, + { + "name": "1.7B-4-ffn_up", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.0.feed_forward.w3", + "module_b": "language_model.model.layers.0.feed_forward.w1", + "distinct_weights": true, + "rows": 64, + "n": 7680, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 1966080 + }, + { + "name": "1.7B-4-ffn_down", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.0.feed_forward.w2", + "module_b": "language_model.model.layers.1.feed_forward.w2", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 7680, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "1.7B-4-ffn_down", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.0.feed_forward.w2", + "module_b": "language_model.model.layers.1.feed_forward.w2", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 7680, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "1.7B-4-attention_q", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.2.self_attn.q_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "1.7B-4-attention_q", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.2.self_attn.q_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "1.7B-4-attention_k", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 1, + "n": 512, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 2048 + }, + { + "name": "1.7B-4-attention_k", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 64, + "n": 512, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 131072 + }, + { + "name": "1.7B-4-attention_v", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.2.self_attn.v_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 1, + "n": 512, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 2048 + }, + { + "name": "1.7B-4-attention_v", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.2.self_attn.v_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 64, + "n": 512, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 131072 + }, + { + "name": "1.7B-4-attention_gate", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.2.self_attn.gate_proj", + "module_b": "language_model.model.layers.10.self_attn.gate_proj", + "distinct_weights": true, + "rows": 1, + "n": 16, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 64 + }, + { + "name": "1.7B-4-attention_gate", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.2.self_attn.gate_proj", + "module_b": "language_model.model.layers.10.self_attn.gate_proj", + "distinct_weights": true, + "rows": 64, + "n": 16, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 4096 + }, + { + "name": "1.7B-4-attention_out", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.2.self_attn.o_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "1.7B-4-attention_out", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.layers.2.self_attn.o_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "1.7B-4-vocabulary_head", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.embed_tokens", + "module_b": "language_model.model.embed_tokens", + "distinct_weights": false, + "rows": 1, + "n": 128000, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 512000 + }, + { + "name": "1.7B-4-vocabulary_head", + "checkpoint": "1.7B-4", + "module_a": "language_model.model.embed_tokens", + "module_b": "language_model.model.embed_tokens", + "distinct_weights": false, + "rows": 64, + "n": 128000, + "k": 2048, + "bits": 4, + "group": 64, + "exit_status": 0, + "equal_bytes": 32768000 + }, + { + "name": "1.7B-6-conv_in", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.0.conv.in_proj", + "module_b": "language_model.model.layers.1.conv.in_proj", + "distinct_weights": true, + "rows": 1, + "n": 6144, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 24576 + }, + { + "name": "1.7B-6-conv_in", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.0.conv.in_proj", + "module_b": "language_model.model.layers.1.conv.in_proj", + "distinct_weights": true, + "rows": 64, + "n": 6144, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 1572864 + }, + { + "name": "1.7B-6-conv_out", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.0.conv.out_proj", + "module_b": "language_model.model.layers.1.conv.out_proj", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "1.7B-6-conv_out", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.0.conv.out_proj", + "module_b": "language_model.model.layers.1.conv.out_proj", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "1.7B-6-ffn_gate", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.0.feed_forward.w1", + "module_b": "language_model.model.layers.0.feed_forward.w3", + "distinct_weights": true, + "rows": 1, + "n": 7680, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 30720 + }, + { + "name": "1.7B-6-ffn_gate", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.0.feed_forward.w1", + "module_b": "language_model.model.layers.0.feed_forward.w3", + "distinct_weights": true, + "rows": 64, + "n": 7680, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 1966080 + }, + { + "name": "1.7B-6-ffn_up", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.0.feed_forward.w3", + "module_b": "language_model.model.layers.0.feed_forward.w1", + "distinct_weights": true, + "rows": 1, + "n": 7680, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 30720 + }, + { + "name": "1.7B-6-ffn_up", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.0.feed_forward.w3", + "module_b": "language_model.model.layers.0.feed_forward.w1", + "distinct_weights": true, + "rows": 64, + "n": 7680, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 1966080 + }, + { + "name": "1.7B-6-ffn_down", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.0.feed_forward.w2", + "module_b": "language_model.model.layers.1.feed_forward.w2", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 7680, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "1.7B-6-ffn_down", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.0.feed_forward.w2", + "module_b": "language_model.model.layers.1.feed_forward.w2", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 7680, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "1.7B-6-attention_q", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.2.self_attn.q_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "1.7B-6-attention_q", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.2.self_attn.q_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "1.7B-6-attention_k", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 1, + "n": 512, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 2048 + }, + { + "name": "1.7B-6-attention_k", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 64, + "n": 512, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 131072 + }, + { + "name": "1.7B-6-attention_v", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.2.self_attn.v_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 1, + "n": 512, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 2048 + }, + { + "name": "1.7B-6-attention_v", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.2.self_attn.v_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 64, + "n": 512, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 131072 + }, + { + "name": "1.7B-6-attention_gate", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.2.self_attn.gate_proj", + "module_b": "language_model.model.layers.10.self_attn.gate_proj", + "distinct_weights": true, + "rows": 1, + "n": 16, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 64 + }, + { + "name": "1.7B-6-attention_gate", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.2.self_attn.gate_proj", + "module_b": "language_model.model.layers.10.self_attn.gate_proj", + "distinct_weights": true, + "rows": 64, + "n": 16, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 4096 + }, + { + "name": "1.7B-6-attention_out", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.2.self_attn.o_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "1.7B-6-attention_out", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.layers.2.self_attn.o_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "1.7B-6-vocabulary_head", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.embed_tokens", + "module_b": "language_model.model.embed_tokens", + "distinct_weights": false, + "rows": 1, + "n": 128000, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 512000 + }, + { + "name": "1.7B-6-vocabulary_head", + "checkpoint": "1.7B-6", + "module_a": "language_model.model.embed_tokens", + "module_b": "language_model.model.embed_tokens", + "distinct_weights": false, + "rows": 64, + "n": 128000, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 32768000 + }, + { + "name": "1.7B-8-conv_in", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.0.conv.in_proj", + "module_b": "language_model.model.layers.1.conv.in_proj", + "distinct_weights": true, + "rows": 1, + "n": 6144, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 24576 + }, + { + "name": "1.7B-8-conv_in", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.0.conv.in_proj", + "module_b": "language_model.model.layers.1.conv.in_proj", + "distinct_weights": true, + "rows": 64, + "n": 6144, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 1572864 + }, + { + "name": "1.7B-8-conv_out", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.0.conv.out_proj", + "module_b": "language_model.model.layers.1.conv.out_proj", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "1.7B-8-conv_out", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.0.conv.out_proj", + "module_b": "language_model.model.layers.1.conv.out_proj", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "1.7B-8-ffn_gate", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.0.feed_forward.w1", + "module_b": "language_model.model.layers.0.feed_forward.w3", + "distinct_weights": true, + "rows": 1, + "n": 7680, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 30720 + }, + { + "name": "1.7B-8-ffn_gate", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.0.feed_forward.w1", + "module_b": "language_model.model.layers.0.feed_forward.w3", + "distinct_weights": true, + "rows": 64, + "n": 7680, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 1966080 + }, + { + "name": "1.7B-8-ffn_up", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.0.feed_forward.w3", + "module_b": "language_model.model.layers.0.feed_forward.w1", + "distinct_weights": true, + "rows": 1, + "n": 7680, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 30720 + }, + { + "name": "1.7B-8-ffn_up", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.0.feed_forward.w3", + "module_b": "language_model.model.layers.0.feed_forward.w1", + "distinct_weights": true, + "rows": 64, + "n": 7680, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 1966080 + }, + { + "name": "1.7B-8-ffn_down", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.0.feed_forward.w2", + "module_b": "language_model.model.layers.1.feed_forward.w2", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 7680, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "1.7B-8-ffn_down", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.0.feed_forward.w2", + "module_b": "language_model.model.layers.1.feed_forward.w2", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 7680, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "1.7B-8-attention_q", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.2.self_attn.q_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "1.7B-8-attention_q", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.2.self_attn.q_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "1.7B-8-attention_k", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 1, + "n": 512, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 2048 + }, + { + "name": "1.7B-8-attention_k", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 64, + "n": 512, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 131072 + }, + { + "name": "1.7B-8-attention_v", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.2.self_attn.v_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 1, + "n": 512, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 2048 + }, + { + "name": "1.7B-8-attention_v", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.2.self_attn.v_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 64, + "n": 512, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 131072 + }, + { + "name": "1.7B-8-attention_gate", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.2.self_attn.gate_proj", + "module_b": "language_model.model.layers.10.self_attn.gate_proj", + "distinct_weights": true, + "rows": 1, + "n": 16, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 64 + }, + { + "name": "1.7B-8-attention_gate", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.2.self_attn.gate_proj", + "module_b": "language_model.model.layers.10.self_attn.gate_proj", + "distinct_weights": true, + "rows": 64, + "n": 16, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 4096 + }, + { + "name": "1.7B-8-attention_out", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.2.self_attn.o_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "1.7B-8-attention_out", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.layers.2.self_attn.o_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "1.7B-8-vocabulary_head", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.embed_tokens", + "module_b": "language_model.model.embed_tokens", + "distinct_weights": false, + "rows": 1, + "n": 128000, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 512000 + }, + { + "name": "1.7B-8-vocabulary_head", + "checkpoint": "1.7B-8", + "module_a": "language_model.model.embed_tokens", + "module_b": "language_model.model.embed_tokens", + "distinct_weights": false, + "rows": 64, + "n": 128000, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 32768000 + }, + { + "name": "VL-2B-4-conv_in", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.0.conv.in_proj", + "module_b": "language_model.model.layers.1.conv.in_proj", + "distinct_weights": true, + "rows": 1, + "n": 6144, + "k": 2048, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 24576 + }, + { + "name": "VL-2B-4-conv_in", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.0.conv.in_proj", + "module_b": "language_model.model.layers.1.conv.in_proj", + "distinct_weights": true, + "rows": 64, + "n": 6144, + "k": 2048, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 1572864 + }, + { + "name": "VL-2B-4-conv_out", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.0.conv.out_proj", + "module_b": "language_model.model.layers.1.conv.out_proj", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 2048, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "VL-2B-4-conv_out", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.0.conv.out_proj", + "module_b": "language_model.model.layers.1.conv.out_proj", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 2048, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "VL-2B-4-ffn_gate", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.0.feed_forward.w1", + "module_b": "language_model.model.layers.0.feed_forward.w3", + "distinct_weights": true, + "rows": 1, + "n": 7680, + "k": 2048, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 30720 + }, + { + "name": "VL-2B-4-ffn_gate", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.0.feed_forward.w1", + "module_b": "language_model.model.layers.0.feed_forward.w3", + "distinct_weights": true, + "rows": 64, + "n": 7680, + "k": 2048, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 1966080 + }, + { + "name": "VL-2B-4-ffn_up", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.0.feed_forward.w3", + "module_b": "language_model.model.layers.0.feed_forward.w1", + "distinct_weights": true, + "rows": 1, + "n": 7680, + "k": 2048, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 30720 + }, + { + "name": "VL-2B-4-ffn_up", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.0.feed_forward.w3", + "module_b": "language_model.model.layers.0.feed_forward.w1", + "distinct_weights": true, + "rows": 64, + "n": 7680, + "k": 2048, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 1966080 + }, + { + "name": "VL-2B-4-ffn_down", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.0.feed_forward.w2", + "module_b": "language_model.model.layers.1.feed_forward.w2", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 7680, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "VL-2B-4-ffn_down", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.0.feed_forward.w2", + "module_b": "language_model.model.layers.1.feed_forward.w2", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 7680, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "VL-2B-4-attention_q", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.2.self_attn.q_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 2048, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "VL-2B-4-attention_q", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.2.self_attn.q_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 2048, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "VL-2B-4-attention_k", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 1, + "n": 512, + "k": 2048, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 2048 + }, + { + "name": "VL-2B-4-attention_k", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 64, + "n": 512, + "k": 2048, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 131072 + }, + { + "name": "VL-2B-4-attention_v", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.2.self_attn.v_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 1, + "n": 512, + "k": 2048, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 2048 + }, + { + "name": "VL-2B-4-attention_v", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.2.self_attn.v_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 64, + "n": 512, + "k": 2048, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 131072 + }, + { + "name": "VL-2B-4-attention_gate", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.2.self_attn.gate_proj", + "module_b": "language_model.model.layers.10.self_attn.gate_proj", + "distinct_weights": true, + "rows": 1, + "n": 16, + "k": 2048, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 64 + }, + { + "name": "VL-2B-4-attention_gate", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.2.self_attn.gate_proj", + "module_b": "language_model.model.layers.10.self_attn.gate_proj", + "distinct_weights": true, + "rows": 64, + "n": 16, + "k": 2048, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 4096 + }, + { + "name": "VL-2B-4-attention_out", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.2.self_attn.o_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 2048, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "VL-2B-4-attention_out", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.layers.2.self_attn.o_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 2048, + "bits": 4, + "group": 32, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "VL-2B-4-vocabulary_head", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.embed_tokens", + "module_b": "language_model.model.embed_tokens", + "distinct_weights": false, + "rows": 1, + "n": 128000, + "k": 2048, + "bits": 8, + "group": 32, + "exit_status": 0, + "equal_bytes": 512000 + }, + { + "name": "VL-2B-4-vocabulary_head", + "checkpoint": "VL-2B-4", + "module_a": "language_model.model.embed_tokens", + "module_b": "language_model.model.embed_tokens", + "distinct_weights": false, + "rows": 64, + "n": 128000, + "k": 2048, + "bits": 8, + "group": 32, + "exit_status": 0, + "equal_bytes": 32768000 + }, + { + "name": "VL-2B-6-conv_in", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.0.conv.in_proj", + "module_b": "language_model.model.layers.1.conv.in_proj", + "distinct_weights": true, + "rows": 1, + "n": 6144, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 24576 + }, + { + "name": "VL-2B-6-conv_in", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.0.conv.in_proj", + "module_b": "language_model.model.layers.1.conv.in_proj", + "distinct_weights": true, + "rows": 64, + "n": 6144, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 1572864 + }, + { + "name": "VL-2B-6-conv_out", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.0.conv.out_proj", + "module_b": "language_model.model.layers.1.conv.out_proj", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "VL-2B-6-conv_out", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.0.conv.out_proj", + "module_b": "language_model.model.layers.1.conv.out_proj", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "VL-2B-6-ffn_gate", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.0.feed_forward.w1", + "module_b": "language_model.model.layers.0.feed_forward.w3", + "distinct_weights": true, + "rows": 1, + "n": 7680, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 30720 + }, + { + "name": "VL-2B-6-ffn_gate", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.0.feed_forward.w1", + "module_b": "language_model.model.layers.0.feed_forward.w3", + "distinct_weights": true, + "rows": 64, + "n": 7680, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 1966080 + }, + { + "name": "VL-2B-6-ffn_up", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.0.feed_forward.w3", + "module_b": "language_model.model.layers.0.feed_forward.w1", + "distinct_weights": true, + "rows": 1, + "n": 7680, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 30720 + }, + { + "name": "VL-2B-6-ffn_up", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.0.feed_forward.w3", + "module_b": "language_model.model.layers.0.feed_forward.w1", + "distinct_weights": true, + "rows": 64, + "n": 7680, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 1966080 + }, + { + "name": "VL-2B-6-ffn_down", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.0.feed_forward.w2", + "module_b": "language_model.model.layers.1.feed_forward.w2", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 7680, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "VL-2B-6-ffn_down", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.0.feed_forward.w2", + "module_b": "language_model.model.layers.1.feed_forward.w2", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 7680, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "VL-2B-6-attention_q", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.2.self_attn.q_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "VL-2B-6-attention_q", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.2.self_attn.q_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "VL-2B-6-attention_k", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 1, + "n": 512, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 2048 + }, + { + "name": "VL-2B-6-attention_k", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 64, + "n": 512, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 131072 + }, + { + "name": "VL-2B-6-attention_v", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.2.self_attn.v_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 1, + "n": 512, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 2048 + }, + { + "name": "VL-2B-6-attention_v", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.2.self_attn.v_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 64, + "n": 512, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 131072 + }, + { + "name": "VL-2B-6-attention_gate", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.2.self_attn.gate_proj", + "module_b": "language_model.model.layers.10.self_attn.gate_proj", + "distinct_weights": true, + "rows": 1, + "n": 16, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 64 + }, + { + "name": "VL-2B-6-attention_gate", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.2.self_attn.gate_proj", + "module_b": "language_model.model.layers.10.self_attn.gate_proj", + "distinct_weights": true, + "rows": 64, + "n": 16, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 4096 + }, + { + "name": "VL-2B-6-attention_out", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.2.self_attn.o_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "VL-2B-6-attention_out", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.layers.2.self_attn.o_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "VL-2B-6-vocabulary_head", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.embed_tokens", + "module_b": "language_model.model.embed_tokens", + "distinct_weights": false, + "rows": 1, + "n": 128000, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 512000 + }, + { + "name": "VL-2B-6-vocabulary_head", + "checkpoint": "VL-2B-6", + "module_a": "language_model.model.embed_tokens", + "module_b": "language_model.model.embed_tokens", + "distinct_weights": false, + "rows": 64, + "n": 128000, + "k": 2048, + "bits": 6, + "group": 64, + "exit_status": 0, + "equal_bytes": 32768000 + }, + { + "name": "VL-2B-8-conv_in", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.0.conv.in_proj", + "module_b": "language_model.model.layers.1.conv.in_proj", + "distinct_weights": true, + "rows": 1, + "n": 6144, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 24576 + }, + { + "name": "VL-2B-8-conv_in", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.0.conv.in_proj", + "module_b": "language_model.model.layers.1.conv.in_proj", + "distinct_weights": true, + "rows": 64, + "n": 6144, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 1572864 + }, + { + "name": "VL-2B-8-conv_out", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.0.conv.out_proj", + "module_b": "language_model.model.layers.1.conv.out_proj", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "VL-2B-8-conv_out", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.0.conv.out_proj", + "module_b": "language_model.model.layers.1.conv.out_proj", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "VL-2B-8-ffn_gate", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.0.feed_forward.w1", + "module_b": "language_model.model.layers.0.feed_forward.w3", + "distinct_weights": true, + "rows": 1, + "n": 7680, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 30720 + }, + { + "name": "VL-2B-8-ffn_gate", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.0.feed_forward.w1", + "module_b": "language_model.model.layers.0.feed_forward.w3", + "distinct_weights": true, + "rows": 64, + "n": 7680, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 1966080 + }, + { + "name": "VL-2B-8-ffn_up", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.0.feed_forward.w3", + "module_b": "language_model.model.layers.0.feed_forward.w1", + "distinct_weights": true, + "rows": 1, + "n": 7680, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 30720 + }, + { + "name": "VL-2B-8-ffn_up", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.0.feed_forward.w3", + "module_b": "language_model.model.layers.0.feed_forward.w1", + "distinct_weights": true, + "rows": 64, + "n": 7680, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 1966080 + }, + { + "name": "VL-2B-8-ffn_down", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.0.feed_forward.w2", + "module_b": "language_model.model.layers.1.feed_forward.w2", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 7680, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "VL-2B-8-ffn_down", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.0.feed_forward.w2", + "module_b": "language_model.model.layers.1.feed_forward.w2", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 7680, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "VL-2B-8-attention_q", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.2.self_attn.q_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "VL-2B-8-attention_q", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.2.self_attn.q_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "VL-2B-8-attention_k", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 1, + "n": 512, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 2048 + }, + { + "name": "VL-2B-8-attention_k", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 64, + "n": 512, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 131072 + }, + { + "name": "VL-2B-8-attention_v", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.2.self_attn.v_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 1, + "n": 512, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 2048 + }, + { + "name": "VL-2B-8-attention_v", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.2.self_attn.v_proj", + "module_b": "language_model.model.layers.10.self_attn.k_proj", + "distinct_weights": true, + "rows": 64, + "n": 512, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 131072 + }, + { + "name": "VL-2B-8-attention_gate", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.2.self_attn.gate_proj", + "module_b": "language_model.model.layers.10.self_attn.gate_proj", + "distinct_weights": true, + "rows": 1, + "n": 16, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 64 + }, + { + "name": "VL-2B-8-attention_gate", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.2.self_attn.gate_proj", + "module_b": "language_model.model.layers.10.self_attn.gate_proj", + "distinct_weights": true, + "rows": 64, + "n": 16, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 4096 + }, + { + "name": "VL-2B-8-attention_out", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.2.self_attn.o_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 1, + "n": 2048, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 8192 + }, + { + "name": "VL-2B-8-attention_out", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.layers.2.self_attn.o_proj", + "module_b": "language_model.model.layers.0.conv.out_proj", + "distinct_weights": true, + "rows": 64, + "n": 2048, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 524288 + }, + { + "name": "VL-2B-8-vocabulary_head", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.embed_tokens", + "module_b": "language_model.model.embed_tokens", + "distinct_weights": false, + "rows": 1, + "n": 128000, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 512000 + }, + { + "name": "VL-2B-8-vocabulary_head", + "checkpoint": "VL-2B-8", + "module_a": "language_model.model.embed_tokens", + "module_b": "language_model.model.embed_tokens", + "distinct_weights": false, + "rows": 64, + "n": 128000, + "k": 2048, + "bits": 8, + "group": 64, + "exit_status": 0, + "equal_bytes": 32768000 + } + ], + "scope": "All 66 projection/layout classes at widths1 and64. The12 vocabulary-head cases use the same tied matrix on both branches; other120 use different stored matrices." + }, + "whole_model": { + "identical_bytes": 394752000, + "cells": [ + { + "checkpoint": "LFM3-1.7B-MLX-4bit", + "rows": 257, + "columns": 128000, + "equal_bytes": 65792000, + "baseline_sha256": "f1dbd42c2556fcb05031a00b209c61fadee45e74ee1ff94628d4802479fcb518", + "candidate_sha256": "f1dbd42c2556fcb05031a00b209c61fadee45e74ee1ff94628d4802479fcb518", + "identical": true + }, + { + "checkpoint": "LFM3-1.7B-MLX-6bit", + "rows": 257, + "columns": 128000, + "equal_bytes": 65792000, + "baseline_sha256": "08bf833a94fa0403bc65c70677d20372b5ab0616fb0a58d26119c486069ef59c", + "candidate_sha256": "08bf833a94fa0403bc65c70677d20372b5ab0616fb0a58d26119c486069ef59c", + "identical": true + }, + { + "checkpoint": "LFM3-1.7B-MLX-8bit", + "rows": 257, + "columns": 128000, + "equal_bytes": 65792000, + "baseline_sha256": "1345bd80acaa16fe08a00a14d839e3e37cf9d0d8cd7e98c01b68cd1b8d2f51d4", + "candidate_sha256": "1345bd80acaa16fe08a00a14d839e3e37cf9d0d8cd7e98c01b68cd1b8d2f51d4", + "identical": true + }, + { + "checkpoint": "LFM3-VL-2B-MLX-4bit", + "rows": 257, + "columns": 128000, + "equal_bytes": 65792000, + "baseline_sha256": "a4dde0a00ee9acb294c2b5a791e7e73cd05c5d6904ffe171e439ad1b01e25c5d", + "candidate_sha256": "a4dde0a00ee9acb294c2b5a791e7e73cd05c5d6904ffe171e439ad1b01e25c5d", + "identical": true + }, + { + "checkpoint": "LFM3-VL-2B-MLX-6bit", + "rows": 257, + "columns": 128000, + "equal_bytes": 65792000, + "baseline_sha256": "93ea83c90c5623f4e17502a526e2358f9e1ba3f2d03999295745257dc95bbca6", + "candidate_sha256": "93ea83c90c5623f4e17502a526e2358f9e1ba3f2d03999295745257dc95bbca6", + "identical": true + }, + { + "checkpoint": "LFM3-VL-2B-MLX-8bit", + "rows": 257, + "columns": 128000, + "equal_bytes": 65792000, + "baseline_sha256": "5a013f3eed97f143429f09a81a829b07dc32360734cd54855e18e514160e0a48", + "candidate_sha256": "5a013f3eed97f143429f09a81a829b07dc32360734cd54855e18e514160e0a48", + "identical": true + } + ], + "scope": "Six full 257-row by128000-column teacher-forced arrays, developer-enabled dual versus plain from the same immutable proof binary. Temporary candidate flag was removed after qualification." + }, + "served": { + "hardware": "M5 Ultra, 256 GiB, macOS27.0 (26A428)", + "counts": { + "passed": 303, + "failed": 0, + "skipped": 0 + }, + "cells": [ + { + "model": "LFM3-1.7B-MLX-4bit", + "binary_sha256": "b804c6ed1859710a7766f66480d569f163e3287b2eb1f8fc6a7ea29434e0562c", + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + } + ], + "logs": [ + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-bhpbywlk/server-pvb2chlc.log", + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-bhpbywlk/server-9ahponnk.log" + ], + "speed_qualified": false, + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6 + }, + { + "model": "LFM3-1.7B-MLX-6bit", + "binary_sha256": "b804c6ed1859710a7766f66480d569f163e3287b2eb1f8fc6a7ea29434e0562c", + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + } + ], + "logs": [ + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-qfemmhfi/server-7hblpn0o.log", + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-qfemmhfi/server-br30tn60.log" + ], + "speed_qualified": false, + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6 + }, + { + "model": "LFM3-1.7B-MLX-8bit", + "binary_sha256": "b804c6ed1859710a7766f66480d569f163e3287b2eb1f8fc6a7ea29434e0562c", + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + } + ], + "logs": [ + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-d8qmv_9q/server-q8rkjqnx.log", + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-d8qmv_9q/server-kk_qp9qg.log" + ], + "speed_qualified": false, + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6 + }, + { + "model": "LFM3-VL-2B-MLX-4bit", + "binary_sha256": "b804c6ed1859710a7766f66480d569f163e3287b2eb1f8fc6a7ea29434e0562c", + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + }, + { + "name": "blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef" + } + }, + { + "name": "red_blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40" + } + }, + { + "name": "blue_red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + }, + { + "name": "red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + }, + { + "name": "blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef" + } + }, + { + "name": "blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef" + } + }, + { + "name": "red_blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40" + } + }, + { + "name": "red_blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40" + } + }, + { + "name": "blue_red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf" + } + }, + { + "name": "blue_red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red" + ], + "observed": [ + "red", + "red", + "red" + ] + } + }, + { + "name": "blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue" + ], + "observed": [ + "blue", + "blue", + "blue" + ] + } + }, + { + "name": "red_blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red", + "blue" + ], + "observed": [ + "red", + "blue" + ] + } + }, + { + "name": "blue_red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue", + "red" + ], + "observed": [ + "blue", + "red" + ] + } + }, + { + "name": "images/concurrent_equals_solos", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + }, + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf" + } + ] + }, + { + "name": "images/corrupt_png_rejected", + "status": "passed", + "evidence": { + "status": 400, + "complete": false + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "red/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 2, + "server_disconnections_after": 3, + "server_confirmed_cancel": true + } + }, + { + "name": "red/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + }, + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + } + ], + "logs": [ + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-oeilhg8y/server-hgv7fucx.log", + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-oeilhg8y/server-jdwykovi.log" + ], + "speed_qualified": false, + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9 + }, + { + "model": "LFM3-VL-2B-MLX-6bit", + "binary_sha256": "b804c6ed1859710a7766f66480d569f163e3287b2eb1f8fc6a7ea29434e0562c", + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + }, + { + "name": "red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + }, + { + "name": "blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0" + } + }, + { + "name": "red_blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5" + } + }, + { + "name": "blue_red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + }, + { + "name": "red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + }, + { + "name": "red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + }, + { + "name": "blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0" + } + }, + { + "name": "blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0" + } + }, + { + "name": "red_blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5" + } + }, + { + "name": "red_blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5" + } + }, + { + "name": "blue_red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603" + } + }, + { + "name": "blue_red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + ] + }, + { + "name": "red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red" + ], + "observed": [ + "red", + "red" + ] + } + }, + { + "name": "blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue" + ], + "observed": [ + "blue", + "blue" + ] + } + }, + { + "name": "red_blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red", + "blue" + ], + "observed": [ + "red", + "blue" + ] + } + }, + { + "name": "blue_red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue", + "red" + ], + "observed": [ + "blue", + "red", + "blue" + ] + } + }, + { + "name": "images/concurrent_equals_solos", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + }, + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603" + } + ] + }, + { + "name": "images/corrupt_png_rejected", + "status": "passed", + "evidence": { + "status": 400, + "complete": false + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "red/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 2, + "server_disconnections_after": 3, + "server_confirmed_cancel": true + } + }, + { + "name": "red/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + }, + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + } + ], + "logs": [ + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-h58vd74f/server-90l0u7no.log", + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-h58vd74f/server-yxg49leq.log" + ], + "speed_qualified": false, + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9 + }, + { + "model": "LFM3-VL-2B-MLX-8bit", + "binary_sha256": "b804c6ed1859710a7766f66480d569f163e3287b2eb1f8fc6a7ea29434e0562c", + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + }, + { + "name": "blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6" + } + }, + { + "name": "red_blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f" + } + }, + { + "name": "blue_red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + }, + { + "name": "red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + }, + { + "name": "blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6" + } + }, + { + "name": "blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6" + } + }, + { + "name": "red_blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f" + } + }, + { + "name": "red_blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f" + } + }, + { + "name": "blue_red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005" + } + }, + { + "name": "blue_red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red" + ], + "observed": [ + "red", + "red" + ] + } + }, + { + "name": "blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue" + ], + "observed": [ + "blue", + "blue", + "blue" + ] + } + }, + { + "name": "red_blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red", + "blue" + ], + "observed": [ + "red", + "blue" + ] + } + }, + { + "name": "blue_red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue", + "red" + ], + "observed": [ + "blue", + "red" + ] + } + }, + { + "name": "images/concurrent_equals_solos", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + }, + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005" + } + ] + }, + { + "name": "images/corrupt_png_rejected", + "status": "passed", + "evidence": { + "status": 400, + "complete": false + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "red/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 2, + "server_disconnections_after": 3, + "server_confirmed_cancel": true + } + }, + { + "name": "red/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + }, + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + } + ], + "logs": [ + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-wjjoncrc/server-m28d5pt2.log", + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-wjjoncrc/server-88291fny.log" + ], + "speed_qualified": false, + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9 + } + ], + "lifecycle": "One owned server process at a time; all owned servers stopped after testing." + }, + "abba": { + "hardware": "M5 Ultra, 256 GiB, macOS27.0 (26A428)", + "settings": { + "context": 70000, + "parallel": 4, + "prompt_cache_gib": 0, + "temperature": 0, + "thinking": false, + "output_tokens": 256, + "repetitions_per_leg": 3 + }, + "summary": { + "baseline_sha256": "d1656249bdb581b147faacbd3b3c557c00b6c45b710eeddce517a67178b6ba2b", + "candidate_sha256": "b804c6ed1859710a7766f66480d569f163e3287b2eb1f8fc6a7ea29434e0562c", + "order": [ + "old", + "new", + "new", + "old" + ], + "counts": { + "alone_equal": 240, + "alone_unequal": 0, + "alone_failed": 0, + "serial_equal": 32, + "serial_unequal": 0, + "serial_failed": 0 + }, + "full_hash_requests": 240, + "hash_groups": [ + { + "family": "text", + "prompt": "long-code-chat", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7" + }, + { + "family": "text", + "prompt": "long-prose-chat", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a" + }, + { + "family": "vl", + "prompt": "long-code-chat", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee" + }, + { + "family": "vl", + "prompt": "long-prose-chat", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa" + } + ], + "cells": [ + { + "family": "text", + "mode": "four_stream", + "prompt": "long-code-chat", + "old_run_medians": [ + 682.4, + 682.6 + ], + "new_run_medians": [ + 836.1, + 837.2 + ], + "old_median": 682.5, + "new_median": 836.6500000000001, + "rate_ratio": 1.225860805860806 + }, + { + "family": "text", + "mode": "four_stream", + "prompt": "long-prose-chat", + "old_run_medians": [ + 703.4, + 731.1 + ], + "new_run_medians": [ + 855.5, + 857.9 + ], + "old_median": 717.25, + "new_median": 856.7, + "rate_ratio": 1.1944231439525967 + }, + { + "family": "text", + "mode": "solo", + "prompt": "long-code-chat", + "old_run_medians": [ + 309.5521887347931, + 308.9845434949348 + ], + "new_run_medians": [ + 316.93726744380007, + 317.8355443591904 + ], + "old_median": 309.26836611486397, + "new_median": 317.3864059014952, + "rate_ratio": 1.0262491760428423 + }, + { + "family": "text", + "mode": "solo", + "prompt": "long-prose-chat", + "old_run_medians": [ + 322.6483820721299, + 322.53863627043455 + ], + "new_run_medians": [ + 332.1536092548651, + 333.4991928050717 + ], + "old_median": 322.5935091712822, + "new_median": 332.8264010299684, + "rate_ratio": 1.03172069979019 + }, + { + "family": "vl", + "mode": "four_stream", + "prompt": "long-code-chat", + "old_run_medians": [ + 661.4, + 650.6 + ], + "new_run_medians": [ + 813.4, + 813.4 + ], + "old_median": 656.0, + "new_median": 813.4, + "rate_ratio": 1.239939024390244 + }, + { + "family": "vl", + "mode": "four_stream", + "prompt": "long-prose-chat", + "old_run_medians": [ + 646.9, + 669.4 + ], + "new_run_medians": [ + 810.7, + 809.5 + ], + "old_median": 658.15, + "new_median": 810.1, + "rate_ratio": 1.2308744207247588 + }, + { + "family": "vl", + "mode": "solo", + "prompt": "long-code-chat", + "old_run_medians": [ + 293.25580648869806, + 292.12128843744097 + ], + "new_run_medians": [ + 301.64586239108235, + 301.7186871597126 + ], + "old_median": 292.68854746306954, + "new_median": 301.68227477539745, + "rate_ratio": 1.030727978222184 + }, + { + "family": "vl", + "mode": "solo", + "prompt": "long-prose-chat", + "old_run_medians": [ + 291.50744489745637, + 290.28747891593963 + ], + "new_run_medians": [ + 299.8096793417327, + 299.99547068513067 + ], + "old_median": 290.897461906698, + "new_median": 299.9025750134317, + "rate_ratio": 1.0309563137736208 + } + ] + }, + "raw_cells": { + "text-4-0-old.concurrent.json": { + "label": "text-4-0-old", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 308.4, + "per_stream_tps": 308.4, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 308.368, + "prefill_tps": 2527.008, + "itl_mean_ms": 3.243, + "itl_p50_ms": 3.292, + "itl_p95_ms": 3.757, + "round_gap_p50_ms": 3.315, + "round_gap_p95_ms": 3.757, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2527.008, + "itl_mean_ms": 3.243, + "itl_p50_ms": 3.292, + "itl_p95_ms": 3.757, + "round_gap_p50_ms": 3.315, + "round_gap_p95_ms": 3.757, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 301.8986888358173, + "makespan_s": 0.8479665843769908, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 308.3676777809419, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8479665843769908, + "end_to_end_tps": 301.8986888358173, + "timing_valid": true, + "per_stream_tps": [ + 308.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.020980375353246927, + "decode_tps": 308.3676777809419, + "prefill_tps": 2526.170247559356, + "itl_mean_ms": 3.2428820270533656, + "itl_p50_ms": 3.2917498610913754, + "itl_p95_ms": 3.767124842852354, + "round_gap_p50_ms": 3.3146669156849384, + "round_gap_p95_ms": 3.767124842852354, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.2428820270533656, + "itl_p50_ms": 3.2917498610913754, + "itl_p95_ms": 3.767124842852354, + "round_gap_p50_ms": 3.3146669156849384, + "round_gap_p95_ms": 3.767124842852354, + "round_gap_mean_ms": 3.3210237626450128, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2526.170247559356 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 308.2753492876164, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8482111250050366, + "end_to_end_tps": 301.81165095951775, + "timing_valid": true, + "per_stream_tps": [ + 308.3 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.020973416045308113, + "decode_tps": 308.2753492876164, + "prefill_tps": 2527.0084704134993, + "itl_mean_ms": 3.2438532704962233, + "itl_p50_ms": 3.298667259514332, + "itl_p95_ms": 3.7559587508440018, + "round_gap_p50_ms": 3.317417111247778, + "round_gap_p95_ms": 3.7559587508440018, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.2438532704962233, + "itl_p50_ms": 3.298667259514332, + "itl_p95_ms": 3.7559587508440018, + "round_gap_p50_ms": 3.317417111247778, + "round_gap_p95_ms": 3.7559587508440018, + "round_gap_mean_ms": 3.3220184095443255, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2527.0084704134993 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 309.38839075058536, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8452341249212623, + "end_to_end_tps": 302.8746621225778, + "timing_valid": true, + "per_stream_tps": [ + 309.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02095987508073449, + "decode_tps": 309.38839075058536, + "prefill_tps": 2528.641024617344, + "itl_mean_ms": 3.232183332975004, + "itl_p50_ms": 3.284416627138853, + "itl_p95_ms": 3.7571247667074203, + "round_gap_p50_ms": 3.3019580878317356, + "round_gap_p95_ms": 3.7571247667074203, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.232183332975004, + "itl_p50_ms": 3.284416627138853, + "itl_p95_ms": 3.7571247667074203, + "round_gap_p50_ms": 3.3019580878317356, + "round_gap_p95_ms": 3.7571247667074203, + "round_gap_mean_ms": 3.3100672687093415, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2528.641024617344 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 682.4, + "per_stream_tps": 173.1, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.053, + "decode_tps": 173.107, + "prefill_tps": 1035.502, + "itl_mean_ms": 5.777, + "itl_p50_ms": 6.155, + "itl_p95_ms": 6.985, + "round_gap_p50_ms": 6.303, + "round_gap_p95_ms": 6.985, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2523.066, + "itl_mean_ms": 5.74, + "itl_p50_ms": 6.136, + "itl_p95_ms": 6.985, + "round_gap_p50_ms": 6.302, + "round_gap_p95_ms": 6.986, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 675.1766506190255, + "makespan_s": 1.5166401253081858, + "steady_tps": 711.0, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 682.3782209713012, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5166401253081858, + "end_to_end_tps": 675.1766506190255, + "timing_valid": true, + "steady_tps": 711.019042486052, + "per_stream_tps": [ + 173.0, + 170.6, + 175.4, + 177.9 + ], + "ttft_s": [ + 0.04, + 0.02, + 0.06, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.041787459049373865, + "decode_tps": 172.99153793489967, + "prefill_tps": 1268.323109509434, + "itl_mean_ms": 5.780629572622916, + "itl_p50_ms": 6.161916069686413, + "itl_p95_ms": 6.993874907493591, + "round_gap_p50_ms": 6.196875125169754, + "round_gap_p95_ms": 6.993874907493591, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021169625222682953, + "decode_tps": 170.61855873115314, + "prefill_tps": 2503.5870707437584, + "itl_mean_ms": 5.861027120594301, + "itl_p50_ms": 6.174833979457617, + "itl_p95_ms": 6.9924998097121716, + "round_gap_p50_ms": 6.183916702866554, + "round_gap_p95_ms": 6.9924998097121716, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06221195915713906, + "decode_tps": 175.41868818853885, + "prefill_tps": 851.9262328024281, + "itl_mean_ms": 5.700646894162192, + "itl_p50_ms": 6.12674979493022, + "itl_p95_ms": 7.02174985781312, + "round_gap_p50_ms": 6.154458038508892, + "round_gap_p95_ms": 7.02174985781312, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08291475009173155, + "decode_tps": 177.94796690630338, + "prefill_tps": 639.2107549183253, + "itl_mean_ms": 5.619620259705127, + "itl_p50_ms": 6.132000125944614, + "itl_p95_ms": 6.994999945163727, + "round_gap_p50_ms": 6.1436668038368225, + "round_gap_p95_ms": 6.994999945163727, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.740480961771134, + "itl_p50_ms": 6.136042065918446, + "itl_p95_ms": 6.9924998097121716, + "round_gap_p50_ms": 6.180291995406151, + "round_gap_p95_ms": 6.993874907493591, + "round_gap_mean_ms": 5.855290581006557, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2556.843019673301 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 682.9737817646647, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5163427917286754, + "end_to_end_tps": 675.3090433018842, + "timing_valid": true, + "steady_tps": 711.9796041649313, + "per_stream_tps": [ + 178.2, + 173.2, + 175.7, + 170.7 + ], + "ttft_s": [ + 0.08, + 0.04, + 0.06, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.0846679168753326, + "decode_tps": 178.19243694766786, + "prefill_tps": 625.9750086687344, + "itl_mean_ms": 5.611910455513234, + "itl_p50_ms": 6.08400022611022, + "itl_p95_ms": 6.8747918121516705, + "round_gap_p50_ms": 6.176874972879887, + "round_gap_p95_ms": 6.8747918121516705, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.043475457932800055, + "decode_tps": 173.2224033768365, + "prefill_tps": 1219.078591004654, + "itl_mean_ms": 5.7729253289746305, + "itl_p50_ms": 6.113916169852018, + "itl_p95_ms": 6.882125046104193, + "round_gap_p50_ms": 6.304207723587751, + "round_gap_p95_ms": 6.882125046104193, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06409724988043308, + "decode_tps": 175.67301352488266, + "prefill_tps": 826.868548945019, + "itl_mean_ms": 5.69239395360152, + "itl_p50_ms": 6.086875218898058, + "itl_p95_ms": 6.8787094205617905, + "round_gap_p50_ms": 6.3036666251719, + "round_gap_p95_ms": 6.8787094205617905, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.02230300009250641, + "decode_tps": 170.74962878025426, + "prefill_tps": 2376.361914548325, + "itl_mean_ms": 5.856528105762075, + "itl_p50_ms": 6.14750012755394, + "itl_p95_ms": 6.884833797812462, + "round_gap_p50_ms": 6.302291993051767, + "round_gap_p95_ms": 6.884833797812462, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.733439460962865, + "itl_p50_ms": 6.090375129133463, + "itl_p95_ms": 6.8787094205617905, + "round_gap_p50_ms": 6.302291993051767, + "round_gap_p95_ms": 6.8812500685453415, + "round_gap_mean_ms": 5.848108250182122, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2501.9300342883034 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 671.1292926699487, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.542270832695067, + "end_to_end_tps": 663.9560175112656, + "timing_valid": true, + "steady_tps": 699.0305334042257, + "per_stream_tps": [ + 175.0, + 167.8, + 170.2, + 172.5 + ], + "ttft_s": [ + 0.08, + 0.02, + 0.04, + 0.06 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.08396570896729827, + "decode_tps": 174.95110157231045, + "prefill_tps": 631.2100576753501, + "itl_mean_ms": 5.715882843908141, + "itl_p50_ms": 6.425541825592518, + "itl_p95_ms": 6.983667146414518, + "round_gap_p50_ms": 6.4604999497532845, + "round_gap_p95_ms": 6.983667146414518, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.02165841730311513, + "decode_tps": 167.78884141856486, + "prefill_tps": 2447.085549154001, + "itl_mean_ms": 5.959871893419939, + "itl_p50_ms": 6.4315833151340485, + "itl_p95_ms": 6.984875071793795, + "round_gap_p50_ms": 6.452916655689478, + "round_gap_p95_ms": 6.984875071793795, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04277845798060298, + "decode_tps": 170.15772670368776, + "prefill_tps": 1238.9413387465197, + "itl_mean_ms": 5.876900328724992, + "itl_p50_ms": 6.466499995440245, + "itl_p95_ms": 6.985666695982218, + "round_gap_p50_ms": 6.473416928201914, + "round_gap_p95_ms": 6.985666695982218, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06350741581991315, + "decode_tps": 172.5246003404771, + "prefill_tps": 834.5482069415509, + "itl_mean_ms": 5.796274838640409, + "itl_p50_ms": 6.451417226344347, + "itl_p95_ms": 7.001208141446114, + "round_gap_p50_ms": 6.456874776631594, + "round_gap_p95_ms": 7.001208141446114, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.83723247617337, + "itl_p50_ms": 6.437874864786863, + "itl_p95_ms": 6.984875071793795, + "round_gap_p50_ms": 6.472874898463488, + "round_gap_p95_ms": 6.985666695982218, + "round_gap_mean_ms": 5.953977125696838, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2523.066129519807 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 322.3, + "per_stream_tps": 322.3, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 322.259, + "prefill_tps": 2534.839, + "itl_mean_ms": 3.103, + "itl_p50_ms": 3.303, + "itl_p95_ms": 3.758, + "round_gap_p50_ms": 3.372, + "round_gap_p95_ms": 3.763, + "tokens_per_chunk": 1.103 + }, + "pooled": { + "aggregate_prefill_tps": 2534.839, + "itl_mean_ms": 3.103, + "itl_p50_ms": 3.303, + "itl_p95_ms": 3.758, + "round_gap_p50_ms": 3.372, + "round_gap_p95_ms": 3.763, + "tokens_per_chunk": 1.103 + }, + "timing_valid": true, + "end_to_end_tps": 315.02705188653096, + "makespan_s": 0.8126286249607801, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 322.25937113597524, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8126286249607801, + "end_to_end_tps": 315.02705188653096, + "timing_valid": true, + "per_stream_tps": [ + 322.3 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021289166994392872, + "decode_tps": 322.25937113597524, + "prefill_tps": 2536.5013114051144, + "itl_mean_ms": 3.103090521386441, + "itl_p50_ms": 3.3034170046448708, + "itl_p95_ms": 3.7551657296717167, + "round_gap_p50_ms": 3.3717919141054153, + "round_gap_p95_ms": 3.7554996088147163, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.103090521386441, + "itl_p50_ms": 3.3034170046448708, + "itl_p95_ms": 3.7551657296717167, + "round_gap_p50_ms": 3.3717919141054153, + "round_gap_p95_ms": 3.7554996088147163, + "round_gap_mean_ms": 3.425489536595422, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2536.5013114051144 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 322.7880276943034, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8113605412654579, + "end_to_end_tps": 315.5194108906547, + "timing_valid": true, + "per_stream_tps": [ + 322.8 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02130312519147992, + "decode_tps": 322.7880276943034, + "prefill_tps": 2534.83934937382, + "itl_mean_ms": 3.0980083342714635, + "itl_p50_ms": 3.2949168235063553, + "itl_p95_ms": 3.758207894861698, + "round_gap_p50_ms": 3.358874935656786, + "round_gap_p95_ms": 3.764000255614519, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.0980083342714635, + "itl_p50_ms": 3.2949168235063553, + "itl_p95_ms": 3.758207894861698, + "round_gap_p50_ms": 3.358874935656786, + "round_gap_p95_ms": 3.764000255614519, + "round_gap_mean_ms": 3.4198793300399273, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2534.83934937382 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 322.07595436624626, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8130934997461736, + "end_to_end_tps": 314.84693959540505, + "timing_valid": true, + "per_stream_tps": [ + 322.1 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02130570774897933, + "decode_tps": 322.07595436624626, + "prefill_tps": 2534.532090471715, + "itl_mean_ms": 3.1048576785799336, + "itl_p50_ms": 3.333040978759527, + "itl_p95_ms": 3.761500120162964, + "round_gap_p50_ms": 3.374207764863968, + "round_gap_p95_ms": 3.763291984796524, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.1048576785799336, + "itl_p50_ms": 3.333040978759527, + "itl_p95_ms": 3.761500120162964, + "round_gap_p50_ms": 3.374207764863968, + "round_gap_p95_ms": 3.763291984796524, + "round_gap_mean_ms": 3.4274402945362903, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2534.532090471715 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 703.4, + "per_stream_tps": 180.3, + "ttft_s_max": 0.09, + "request": { + "ttft_s": 0.054, + "decode_tps": 180.313, + "prefill_tps": 1031.154, + "itl_mean_ms": 5.546, + "itl_p50_ms": 5.077, + "itl_p95_ms": 7.122, + "round_gap_p50_ms": 6.269, + "round_gap_p95_ms": 7.158, + "tokens_per_chunk": 1.08 + }, + "pooled": { + "aggregate_prefill_tps": 2518.309, + "itl_mean_ms": 5.563, + "itl_p50_ms": 5.062, + "itl_p95_ms": 7.12, + "round_gap_p50_ms": 6.306, + "round_gap_p95_ms": 7.135, + "tokens_per_chunk": 1.08 + }, + "timing_valid": true, + "end_to_end_tps": 695.5611725526805, + "makespan_s": 1.4721925840713084, + "steady_tps": 734.5, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 703.4482556657692, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.4721925840713084, + "end_to_end_tps": 695.5611725526805, + "timing_valid": true, + "steady_tps": 734.4537871680274, + "per_stream_tps": [ + 181.1, + 175.9, + 183.8, + 178.4 + ], + "ttft_s": [ + 0.06, + 0.02, + 0.08, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.0633647502399981, + "decode_tps": 181.0835133063667, + "prefill_tps": 852.2088352825742, + "itl_mean_ms": 5.522313885682938, + "itl_p50_ms": 5.081584211438894, + "itl_p95_ms": 7.190666161477566, + "round_gap_p50_ms": 6.268375087529421, + "round_gap_p95_ms": 7.192040793597698, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.02152333315461874, + "decode_tps": 175.86882067397312, + "prefill_tps": 2508.905085103513, + "itl_mean_ms": 5.686056210349004, + "itl_p50_ms": 5.076250061392784, + "itl_p95_ms": 7.183166686445475, + "round_gap_p50_ms": 6.322000175714493, + "round_gap_p95_ms": 7.184749934822321, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08420174987986684, + "decode_tps": 183.81099113137475, + "prefill_tps": 641.316838154117, + "itl_mean_ms": 5.440371078165138, + "itl_p50_ms": 5.051415879279375, + "itl_p95_ms": 7.1499161422252655, + "round_gap_p50_ms": 6.259874906390905, + "round_gap_p95_ms": 7.163667120039463, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.042514333967119455, + "decode_tps": 178.4397386507722, + "prefill_tps": 1270.1598487174595, + "itl_mean_ms": 5.604132843733418, + "itl_p50_ms": 5.077666137367487, + "itl_p95_ms": 7.135250139981508, + "round_gap_p50_ms": 6.269624922424555, + "round_gap_p95_ms": 7.164209149777889, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.563218504482625, + "itl_p50_ms": 5.0615002401173115, + "itl_p95_ms": 7.1499161422252655, + "round_gap_p50_ms": 6.306333001703024, + "round_gap_p95_ms": 7.183166686445475, + "round_gap_mean_ms": 6.011104740013005, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2561.888831337787 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 712.2186344374949, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.4558278750628233, + "end_to_end_tps": 703.3798552289785, + "timing_valid": true, + "steady_tps": 744.0323812760112, + "per_stream_tps": [ + 180.7, + 186.2, + 183.4, + 178.1 + ], + "ttft_s": [ + 0.04, + 0.09, + 0.06, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.043958334252238274, + "decode_tps": 180.70520744800967, + "prefill_tps": 1228.4359932781217, + "itl_mean_ms": 5.533874834723332, + "itl_p50_ms": 5.073375068604946, + "itl_p95_ms": 7.006458938121796, + "round_gap_p50_ms": 5.273249931633472, + "round_gap_p95_ms": 7.020959164947271, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08566550025716424, + "decode_tps": 186.2052006351326, + "prefill_tps": 630.3587773128537, + "itl_mean_ms": 5.370419282539218, + "itl_p50_ms": 5.036000162363052, + "itl_p95_ms": 6.99370913207531, + "round_gap_p50_ms": 5.277582909911871, + "round_gap_p95_ms": 7.015665993094444, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06472079223021865, + "decode_tps": 183.40206524497208, + "prefill_tps": 834.3531983959086, + "itl_mean_ms": 5.452501304520696, + "itl_p50_ms": 5.02687506377697, + "itl_p95_ms": 7.018500007688999, + "round_gap_p50_ms": 5.304291844367981, + "round_gap_p95_ms": 7.056749891489744, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.023153916001319885, + "decode_tps": 178.0729158046269, + "prefill_tps": 2332.2188780905026, + "itl_mean_ms": 5.615677125751972, + "itl_p50_ms": 5.067334044724703, + "itl_p95_ms": 7.026041857898235, + "round_gap_p50_ms": 5.307542160153389, + "round_gap_p95_ms": 7.055333815515041, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.493118136883805, + "itl_p50_ms": 5.036000162363052, + "itl_p95_ms": 7.006458938121796, + "round_gap_p50_ms": 5.292791873216629, + "round_gap_p95_ms": 7.020959164947271, + "round_gap_mean_ms": 5.935360698751569, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2518.309242575295 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 698.7828759433579, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.4834294579923153, + "end_to_end_tps": 690.2923455395644, + "timing_valid": true, + "steady_tps": 729.6414329932904, + "per_stream_tps": [ + 177.3, + 174.7, + 179.9, + 182.6 + ], + "ttft_s": [ + 0.04, + 0.02, + 0.07, + 0.09 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.04462445806711912, + "decode_tps": 177.31427800098217, + "prefill_tps": 1210.0987292390025, + "itl_mean_ms": 5.639703758060932, + "itl_p50_ms": 6.039209198206663, + "itl_p95_ms": 7.12037505581975, + "round_gap_p50_ms": 6.385957822203636, + "round_gap_p95_ms": 7.163917180150747, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.023088957648724318, + "decode_tps": 174.70362318244156, + "prefill_tps": 2338.780330474708, + "itl_mean_ms": 5.723979742284499, + "itl_p50_ms": 6.109416950494051, + "itl_p95_ms": 7.121209055185318, + "round_gap_p50_ms": 6.385458633303642, + "round_gap_p95_ms": 7.163333240896463, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06541320914402604, + "decode_tps": 179.92050157463905, + "prefill_tps": 825.5213389868005, + "itl_mean_ms": 5.55801029481432, + "itl_p50_ms": 6.088166031986475, + "itl_p95_ms": 7.1409158408641815, + "round_gap_p50_ms": 6.379582919180393, + "round_gap_p95_ms": 7.153375074267387, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08641408290714025, + "decode_tps": 182.6053050734621, + "prefill_tps": 624.898143720716, + "itl_mean_ms": 5.476292156997849, + "itl_p50_ms": 5.896500311791897, + "itl_p95_ms": 7.123582996428013, + "round_gap_p50_ms": 6.365375127643347, + "round_gap_p95_ms": 7.127667311578989, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.5994964880394, + "itl_p50_ms": 6.039209198206663, + "itl_p95_ms": 7.12037505581975, + "round_gap_p50_ms": 6.381667219102383, + "round_gap_p95_ms": 7.13516678661108, + "round_gap_mean_ms": 6.05030340868664, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2499.592574882864 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + } + ] + } + ] + }, + "text-4-0-old.decode.json": [ + { + "label": "text-4-0-old", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 309.5521887347931, + "decode_tps_all": [ + 309.8, + 309.55, + 309.2 + ], + "ttft_s_median": 0.0210205833427608, + "prefill_tps_median": 2521.338, + "itl_mean_ms_median": 3.23, + "itl_p50_ms_median": 3.286, + "itl_p95_ms_median": 3.743, + "round_gap_p50_ms_median": 3.301, + "round_gap_p95_ms_median": 3.743, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "sample": "Here is a complete Python implementation of a Red-Black Tree with insertion, deletion, searching, rotations, and extensive unit tests. I'll provide detailed exp" + }, + { + "label": "text-4-0-old", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 322.6483820721299, + "decode_tps_all": [ + 322.61, + 322.71, + 322.65 + ], + "ttft_s_median": 0.021303792018443346, + "prefill_tps_median": 2534.76, + "itl_mean_ms_median": 3.099, + "itl_p50_ms_median": 3.31, + "itl_p95_ms_median": 3.76, + "round_gap_p50_ms_median": 3.366, + "round_gap_p95_ms_median": 3.763, + "tokens_per_chunk_median": 1.103, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "sample": "We need to write a detailed 2000-word tutorial explaining how matrix multiplication runs on a GPU, covering memory layout, tiling, shared memory, synchro" + } + ], + "text-4-1-new.concurrent.json": { + "label": "text-4-1-new", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 317.5, + "per_stream_tps": 317.5, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 317.469, + "prefill_tps": 2528.591, + "itl_mean_ms": 3.15, + "itl_p50_ms": 3.205, + "itl_p95_ms": 3.671, + "round_gap_p50_ms": 3.23, + "round_gap_p95_ms": 3.671, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2528.591, + "itl_mean_ms": 3.15, + "itl_p50_ms": 3.205, + "itl_p95_ms": 3.671, + "round_gap_p50_ms": 3.23, + "round_gap_p95_ms": 3.671, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 310.5660548093696, + "makespan_s": 0.824301291257143, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 317.4694243651148, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8242529579438269, + "end_to_end_tps": 310.584266071201, + "timing_valid": true, + "per_stream_tps": [ + 317.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02093224972486496, + "decode_tps": 317.4694243651148, + "prefill_tps": 2531.978200940459, + "itl_mean_ms": 3.1499096393293025, + "itl_p50_ms": 3.2014590688049793, + "itl_p95_ms": 3.667040728032589, + "round_gap_p50_ms": 3.226333297789097, + "round_gap_p95_ms": 3.667040728032589, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.1499096393293025, + "itl_p50_ms": 3.2014590688049793, + "itl_p95_ms": 3.667040728032589, + "round_gap_p50_ms": 3.226333297789097, + "round_gap_p95_ms": 3.667040728032589, + "round_gap_mean_ms": 3.225811076421575, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2531.978200940459 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 317.4689798870941, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.824301291257143, + "end_to_end_tps": 310.5660548093696, + "timing_valid": true, + "per_stream_tps": [ + 317.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021016333252191544, + "decode_tps": 317.4689798870941, + "prefill_tps": 2521.848096145566, + "itl_mean_ms": 3.149914049415612, + "itl_p50_ms": 3.2050837762653828, + "itl_p95_ms": 3.67079209536314, + "round_gap_p50_ms": 3.2297498546540737, + "round_gap_p95_ms": 3.67079209536314, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.149914049415612, + "itl_p50_ms": 3.2050837762653828, + "itl_p95_ms": 3.67079209536314, + "round_gap_p50_ms": 3.2297498546540737, + "round_gap_p95_ms": 3.67079209536314, + "round_gap_mean_ms": 3.225815592775024, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2521.848096145566 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 316.6510815661193, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8263304159045219, + "end_to_end_tps": 309.80343343621934, + "timing_valid": true, + "per_stream_tps": [ + 316.7 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.020960290916264057, + "decode_tps": 316.6510815661193, + "prefill_tps": 2528.5908583871255, + "itl_mean_ms": 3.158050163776851, + "itl_p50_ms": 3.2296250574290752, + "itl_p95_ms": 3.6792918108403683, + "round_gap_p50_ms": 3.2383748330175877, + "round_gap_p95_ms": 3.6792918108403683, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.158050163776851, + "itl_p50_ms": 3.2296250574290752, + "itl_p95_ms": 3.6792918108403683, + "round_gap_p50_ms": 3.2383748330175877, + "round_gap_p95_ms": 3.6792918108403683, + "round_gap_mean_ms": 3.234147758084727, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2528.5908583871255 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 836.1, + "per_stream_tps": 214.4, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.052, + "decode_tps": 214.442, + "prefill_tps": 1064.079, + "itl_mean_ms": 4.664, + "itl_p50_ms": 4.608, + "itl_p95_ms": 5.019, + "round_gap_p50_ms": 4.617, + "round_gap_p95_ms": 5.019, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2566.969, + "itl_mean_ms": 4.663, + "itl_p50_ms": 4.608, + "itl_p95_ms": 5.018, + "round_gap_p50_ms": 4.622, + "round_gap_p95_ms": 5.018, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 824.8147055925709, + "makespan_s": 1.2414909591898322, + "steady_tps": 879.5, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 835.5952376823824, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2424355829134583, + "end_to_end_tps": 824.1875990051442, + "timing_valid": true, + "steady_tps": 878.8823110084487, + "per_stream_tps": [ + 208.9, + 212.5, + 220.0, + 216.2 + ], + "ttft_s": [ + 0.02, + 0.04, + 0.08, + 0.06 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021172000095248222, + "decode_tps": 208.90813655351263, + "prefill_tps": 2503.3062422805842, + "itl_mean_ms": 4.786792972727733, + "itl_p50_ms": 4.617167171090841, + "itl_p95_ms": 5.0171250477433205, + "round_gap_p50_ms": 4.617291036993265, + "round_gap_p95_ms": 5.0171250477433205, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.041625124868005514, + "decode_tps": 212.47136805725867, + "prefill_tps": 1273.269453678867, + "itl_mean_ms": 4.706516502169418, + "itl_p50_ms": 4.605832975357771, + "itl_p95_ms": 5.020374897867441, + "round_gap_p50_ms": 4.617583937942982, + "round_gap_p95_ms": 5.020374897867441, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08249250007793307, + "decode_tps": 219.96239224902337, + "prefill_tps": 642.482649330901, + "itl_mean_ms": 4.5462316979526305, + "itl_p50_ms": 4.60820784792304, + "itl_p95_ms": 5.019000265747309, + "round_gap_p50_ms": 4.621082916855812, + "round_gap_p95_ms": 5.019000265747309, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06216404167935252, + "decode_tps": 216.15140509625442, + "prefill_tps": 852.5829172012103, + "itl_mean_ms": 4.6263867660480384, + "itl_p50_ms": 4.6190838329494, + "itl_p95_ms": 5.017084069550037, + "round_gap_p50_ms": 4.63133305311203, + "round_gap_p95_ms": 5.017084069550037, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.666481984724455, + "itl_p50_ms": 4.616583231836557, + "itl_p95_ms": 5.017583724111319, + "round_gap_p50_ms": 4.6221669763326645, + "round_gap_p95_ms": 5.018084309995174, + "round_gap_mean_ms": 4.759811624418944, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2566.053950422707 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 836.6287835338114, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2408760827966034, + "end_to_end_tps": 825.2234160982274, + "timing_valid": true, + "steady_tps": 880.006451320497, + "per_stream_tps": [ + 209.2, + 212.7, + 216.4, + 220.2 + ], + "ttft_s": [ + 0.02, + 0.04, + 0.06, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021155124995857477, + "decode_tps": 209.16422984210257, + "prefill_tps": 2505.3030889856846, + "itl_mean_ms": 4.780932192635887, + "itl_p50_ms": 4.601041786372662, + "itl_p95_ms": 5.016874987632036, + "round_gap_p50_ms": 4.611458163708448, + "round_gap_p95_ms": 5.016874987632036, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04159566713497043, + "decode_tps": 212.73309042544162, + "prefill_tps": 1274.1711733586235, + "itl_mean_ms": 4.700726144673193, + "itl_p50_ms": 4.609667230397463, + "itl_p95_ms": 5.023208912461996, + "round_gap_p50_ms": 4.615874961018562, + "round_gap_p95_ms": 5.023208912461996, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06199637474492192, + "decode_tps": 216.42396171743155, + "prefill_tps": 854.8886966062671, + "itl_mean_ms": 4.620560459500434, + "itl_p50_ms": 4.606916103512049, + "itl_p95_ms": 5.0180829130113125, + "round_gap_p50_ms": 4.609833937138319, + "round_gap_p95_ms": 5.0180829130113125, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08258766680955887, + "decode_tps": 220.2462915274576, + "prefill_tps": 641.7423090812595, + "itl_mean_ms": 4.540371567960463, + "itl_p50_ms": 4.588041920214891, + "itl_p95_ms": 5.014833994209766, + "round_gap_p50_ms": 4.616457968950272, + "round_gap_p95_ms": 5.014833994209766, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.660647591192494, + "itl_p50_ms": 4.603915847837925, + "itl_p95_ms": 5.017375107854605, + "round_gap_p50_ms": 4.615874961018562, + "round_gap_p95_ms": 5.017417017370462, + "round_gap_mean_ms": 4.753860543016344, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2566.969236325038 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 836.0876178920478, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2414909591898322, + "end_to_end_tps": 824.8147055925709, + "timing_valid": true, + "steady_tps": 879.5214281065635, + "per_stream_tps": [ + 216.3, + 212.6, + 220.1, + 209.1 + ], + "ttft_s": [ + 0.06, + 0.04, + 0.08, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.062059083953499794, + "decode_tps": 216.29838746021036, + "prefill_tps": 854.0248521829992, + "itl_mean_ms": 4.623242973477817, + "itl_p50_ms": 4.612792283296585, + "itl_p95_ms": 5.02866692841053, + "round_gap_p50_ms": 4.618166014552116, + "round_gap_p95_ms": 5.02866692841053, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04158137505874038, + "decode_tps": 212.62177021533628, + "prefill_tps": 1274.6091230780362, + "itl_mean_ms": 4.703187255882749, + "itl_p50_ms": 4.604457877576351, + "itl_p95_ms": 5.023290868848562, + "round_gap_p50_ms": 4.6304999850690365, + "round_gap_p95_ms": 5.023290868848562, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.0824298751540482, + "decode_tps": 220.0873984550488, + "prefill_tps": 642.9707663750735, + "itl_mean_ms": 4.543649509330006, + "itl_p50_ms": 4.600792191922665, + "itl_p95_ms": 5.020792130380869, + "round_gap_p50_ms": 4.6090418472886086, + "round_gap_p95_ms": 5.020792130380869, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021227250341326, + "decode_tps": 209.0627613347365, + "prefill_tps": 2496.7906416413075, + "itl_mean_ms": 4.78325261570075, + "itl_p50_ms": 4.616708029061556, + "itl_p95_ms": 5.018416792154312, + "round_gap_p50_ms": 4.6253749169409275, + "round_gap_p95_ms": 5.018416792154312, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.663333088597831, + "itl_p50_ms": 4.607874900102615, + "itl_p95_ms": 5.021125078201294, + "round_gap_p50_ms": 4.624292254447937, + "round_gap_p95_ms": 5.021916702389717, + "round_gap_mean_ms": 4.756599750369787, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2567.690772755544 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 332.0, + "per_stream_tps": 332.0, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 331.961, + "prefill_tps": 2546.89, + "itl_mean_ms": 3.012, + "itl_p50_ms": 3.231, + "itl_p95_ms": 3.688, + "round_gap_p50_ms": 3.286, + "round_gap_p95_ms": 3.691, + "tokens_per_chunk": 1.103 + }, + "pooled": { + "aggregate_prefill_tps": 2546.89, + "itl_mean_ms": 3.012, + "itl_p50_ms": 3.231, + "itl_p95_ms": 3.688, + "round_gap_p50_ms": 3.286, + "round_gap_p95_ms": 3.691, + "tokens_per_chunk": 1.103 + }, + "timing_valid": true, + "end_to_end_tps": 324.29109747667655, + "makespan_s": 0.7894142083823681, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 332.04074387188365, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7892718748189509, + "end_to_end_tps": 324.34957860207953, + "timing_valid": true, + "per_stream_tps": [ + 332.0 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02124391682446003, + "decode_tps": 332.04074387188365, + "prefill_tps": 2541.9041340730987, + "itl_mean_ms": 3.011678592027987, + "itl_p50_ms": 3.230665810406208, + "itl_p95_ms": 3.682082984596491, + "round_gap_p50_ms": 3.2878746278584003, + "round_gap_p95_ms": 3.683416172862053, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.011678592027987, + "itl_p50_ms": 3.230665810406208, + "itl_p95_ms": 3.682082984596491, + "round_gap_p50_ms": 3.2878746278584003, + "round_gap_p95_ms": 3.683416172862053, + "round_gap_mean_ms": 3.3245802639269986, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2541.9041340730987 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 331.9609029428107, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7894142083823681, + "end_to_end_tps": 324.29109747667655, + "timing_valid": true, + "per_stream_tps": [ + 332.0 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02120233327150345, + "decode_tps": 331.9609029428107, + "prefill_tps": 2546.889500721959, + "itl_mean_ms": 3.0124029400301913, + "itl_p50_ms": 3.225708846002817, + "itl_p95_ms": 3.6877919919788837, + "round_gap_p50_ms": 3.2859579659998417, + "round_gap_p95_ms": 3.6909999325871468, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.0124029400301913, + "itl_p50_ms": 3.225708846002817, + "itl_p95_ms": 3.6877919919788837, + "round_gap_p50_ms": 3.2859579659998417, + "round_gap_p95_ms": 3.6909999325871468, + "round_gap_mean_ms": 3.325379868864497, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2546.889500721959 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 331.16918985548796, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7912421249784529, + "end_to_end_tps": 323.54192467567543, + "timing_valid": true, + "per_stream_tps": [ + 331.2 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021192458923906088, + "decode_tps": 331.16918985548796, + "prefill_tps": 2548.0761903983434, + "itl_mean_ms": 3.0196045726245524, + "itl_p50_ms": 3.233250230550766, + "itl_p95_ms": 3.6952500231564045, + "round_gap_p50_ms": 3.286417108029127, + "round_gap_p95_ms": 3.699416760355234, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.0196045726245524, + "itl_p50_ms": 3.233250230550766, + "itl_p95_ms": 3.6952500231564045, + "round_gap_p50_ms": 3.286417108029127, + "round_gap_p95_ms": 3.699416760355234, + "round_gap_mean_ms": 3.3333297230271035, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2548.0761903983434 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 855.5, + "per_stream_tps": 219.7, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.053, + "decode_tps": 219.709, + "prefill_tps": 1068.041, + "itl_mean_ms": 4.552, + "itl_p50_ms": 4.642, + "itl_p95_ms": 5.048, + "round_gap_p50_ms": 4.682, + "round_gap_p95_ms": 5.055, + "tokens_per_chunk": 1.08 + }, + "pooled": { + "aggregate_prefill_tps": 2577.432, + "itl_mean_ms": 4.553, + "itl_p50_ms": 4.641, + "itl_p95_ms": 5.044, + "round_gap_p50_ms": 4.683, + "round_gap_p95_ms": 5.052, + "tokens_per_chunk": 1.08 + }, + "timing_valid": true, + "end_to_end_tps": 843.2969574941419, + "makespan_s": 1.2142816251143813, + "steady_tps": 901.7, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 856.3735602687286, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2130478327162564, + "end_to_end_tps": 844.1546758358733, + "timing_valid": true, + "steady_tps": 902.6978161069744, + "per_stream_tps": [ + 217.9, + 225.9, + 221.8, + 214.1 + ], + "ttft_s": [ + 0.04, + 0.08, + 0.06, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.04222516668960452, + "decode_tps": 217.89353068267215, + "prefill_tps": 1278.8581841950277, + "itl_mean_ms": 4.589397385351214, + "itl_p50_ms": 4.645875189453363, + "itl_p95_ms": 5.03958435729146, + "round_gap_p50_ms": 4.689207766205072, + "round_gap_p95_ms": 5.042084027081728, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08376608323305845, + "decode_tps": 225.91945065516143, + "prefill_tps": 644.6523212713471, + "itl_mean_ms": 4.426356372149551, + "itl_p50_ms": 4.643624648451805, + "itl_p95_ms": 5.0467089749872684, + "round_gap_p50_ms": 4.6713752672076225, + "round_gap_p95_ms": 5.0515420734882355, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06292216619476676, + "decode_tps": 221.8280063175558, + "prefill_tps": 858.2031304016228, + "itl_mean_ms": 4.507997058624146, + "itl_p50_ms": 4.644832573831081, + "itl_p95_ms": 5.050084087997675, + "round_gap_p50_ms": 4.685165826231241, + "round_gap_p95_ms": 5.057916045188904, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021514040883630514, + "decode_tps": 214.10065507204197, + "prefill_tps": 2509.9887228106563, + "itl_mean_ms": 4.670700328607185, + "itl_p50_ms": 4.649666137993336, + "itl_p95_ms": 5.038416013121605, + "round_gap_p50_ms": 4.674999974668026, + "round_gap_p95_ms": 5.054790992289782, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.548612786183024, + "itl_p50_ms": 4.644542001187801, + "itl_p95_ms": 5.042084027081728, + "round_gap_p50_ms": 4.682499915361404, + "round_gap_p95_ms": 5.050084087997675, + "round_gap_mean_ms": 4.914814663036743, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2575.8417553725017 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 854.9576263603261, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2145862081088126, + "end_to_end_tps": 843.0854830752876, + "timing_valid": true, + "steady_tps": 901.4075226712479, + "per_stream_tps": [ + 213.8, + 217.6, + 221.5, + 225.5 + ], + "ttft_s": [ + 0.02, + 0.04, + 0.06, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021480209194123745, + "decode_tps": 213.8251143270371, + "prefill_tps": 2513.9419971185644, + "itl_mean_ms": 4.67671911761749, + "itl_p50_ms": 4.643084015697241, + "itl_p95_ms": 5.047000013291836, + "round_gap_p50_ms": 4.68145776540041, + "round_gap_p95_ms": 5.07283303886652, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.042201667092740536, + "decode_tps": 217.60940856413748, + "prefill_tps": 1279.5703042093567, + "itl_mean_ms": 4.5953895403620075, + "itl_p50_ms": 4.6394579112529755, + "itl_p95_ms": 5.060082767158747, + "round_gap_p50_ms": 4.68783313408494, + "round_gap_p95_ms": 5.063999909907579, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06288537522777915, + "decode_tps": 221.52532657719738, + "prefill_tps": 858.7052204809919, + "itl_mean_ms": 4.514156532126899, + "itl_p50_ms": 4.646542016416788, + "itl_p95_ms": 5.0505842082202435, + "round_gap_p50_ms": 4.690249916166067, + "round_gap_p95_ms": 5.056084133684635, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08378549991175532, + "decode_tps": 225.50798158142572, + "prefill_tps": 644.5029277962649, + "itl_mean_ms": 4.434432843517439, + "itl_p50_ms": 4.639542195945978, + "itl_p95_ms": 5.047790706157684, + "round_gap_p50_ms": 4.679084289819002, + "round_gap_p95_ms": 5.048333667218685, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.555174508405959, + "itl_p50_ms": 4.64116595685482, + "itl_p95_ms": 5.047000013291836, + "round_gap_p50_ms": 4.68783313408494, + "round_gap_p95_ms": 5.056084133684635, + "round_gap_mean_ms": 4.921904659506438, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2578.0117111850595 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 855.4680742595434, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2142816251143813, + "end_to_end_tps": 843.2969574941419, + "timing_valid": true, + "steady_tps": 901.6739313774285, + "per_stream_tps": [ + 221.6, + 213.9, + 225.7, + 217.7 + ], + "ttft_s": [ + 0.06, + 0.02, + 0.08, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.06299304123967886, + "decode_tps": 221.59267848176944, + "prefill_tps": 857.2375446128768, + "itl_mean_ms": 4.512784478492012, + "itl_p50_ms": 4.627209156751633, + "itl_p95_ms": 5.060417111963034, + "round_gap_p50_ms": 4.693999886512756, + "round_gap_p95_ms": 5.065583623945713, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021404416300356388, + "decode_tps": 213.87301275264053, + "prefill_tps": 2522.84384877624, + "itl_mean_ms": 4.675671732162728, + "itl_p50_ms": 4.627166781574488, + "itl_p95_ms": 5.052125081419945, + "round_gap_p50_ms": 4.682125058025122, + "round_gap_p95_ms": 5.053750239312649, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08369216695427895, + "decode_tps": 225.663392289006, + "prefill_tps": 645.2216732481095, + "itl_mean_ms": 4.431378921749545, + "itl_p50_ms": 4.626541864126921, + "itl_p95_ms": 5.04270801320672, + "round_gap_p50_ms": 4.682417027652264, + "round_gap_p95_ms": 5.047874990850687, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.042274124920368195, + "decode_tps": 217.667656919266, + "prefill_tps": 1277.3771213885525, + "itl_mean_ms": 4.594159803773258, + "itl_p50_ms": 4.64187515899539, + "itl_p95_ms": 5.0490000285208225, + "round_gap_p50_ms": 4.678457975387573, + "round_gap_p95_ms": 5.068750120699406, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.5534987340443855, + "itl_p50_ms": 4.627166781574488, + "itl_p95_ms": 5.043749697506428, + "round_gap_p50_ms": 4.6825832687318325, + "round_gap_p95_ms": 5.052125081419945, + "round_gap_mean_ms": 4.920093971107281, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2577.4323303591286 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + } + ] + } + ] + }, + "text-4-1-new.decode.json": [ + { + "label": "text-4-1-new", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 316.93726744380007, + "decode_tps_all": [ + 316.38, + 317.38, + 316.94 + ], + "ttft_s_median": 0.020918041933327913, + "prefill_tps_median": 2533.698, + "itl_mean_ms_median": 3.155, + "itl_p50_ms_median": 3.217, + "itl_p95_ms_median": 3.668, + "round_gap_p50_ms_median": 3.228, + "round_gap_p95_ms_median": 3.668, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "sample": "Here is a complete Python implementation of a Red-Black Tree with insertion, deletion, searching, rotations, and extensive unit tests. I'll provide detailed exp" + }, + { + "label": "text-4-1-new", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 332.1536092548651, + "decode_tps_all": [ + 331.95, + 332.72, + 332.15 + ], + "ttft_s_median": 0.021217749919742346, + "prefill_tps_median": 2545.039, + "itl_mean_ms_median": 3.011, + "itl_p50_ms_median": 3.231, + "itl_p95_ms_median": 3.68, + "round_gap_p50_ms_median": 3.282, + "round_gap_p95_ms_median": 3.686, + "tokens_per_chunk_median": 1.103, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "sample": "We need to write a detailed 2000-word tutorial explaining how matrix multiplication runs on a GPU, covering memory layout, tiling, shared memory, synchro" + } + ], + "text-4-2-new.concurrent.json": { + "label": "text-4-2-new", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 318.7, + "per_stream_tps": 318.7, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 318.701, + "prefill_tps": 2537.362, + "itl_mean_ms": 3.138, + "itl_p50_ms": 3.19, + "itl_p95_ms": 3.656, + "round_gap_p50_ms": 3.197, + "round_gap_p95_ms": 3.656, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2537.362, + "itl_mean_ms": 3.138, + "itl_p50_ms": 3.19, + "itl_p95_ms": 3.656, + "round_gap_p50_ms": 3.197, + "round_gap_p95_ms": 3.656, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 311.787311481795, + "makespan_s": 0.8210725407116115, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 319.379010298801, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8193982499651611, + "end_to_end_tps": 312.42439193747913, + "timing_valid": true, + "per_stream_tps": [ + 319.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02092166617512703, + "decode_tps": 319.379010298801, + "prefill_tps": 2533.259041433788, + "itl_mean_ms": 3.13107614387192, + "itl_p50_ms": 3.188124857842922, + "itl_p95_ms": 3.654541913419962, + "round_gap_p50_ms": 3.1965416856110096, + "round_gap_p95_ms": 3.654541913419962, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.13107614387192, + "itl_p50_ms": 3.188124857842922, + "itl_p95_ms": 3.654541913419962, + "round_gap_p50_ms": 3.1965416856110096, + "round_gap_p95_ms": 3.654541913419962, + "round_gap_mean_ms": 3.2065237617965443, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2533.259041433788 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 318.7006181689939, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8210725407116115, + "end_to_end_tps": 311.787311481795, + "timing_valid": true, + "per_stream_tps": [ + 318.7 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.020884457975625992, + "decode_tps": 318.7006181689939, + "prefill_tps": 2537.772350225976, + "itl_mean_ms": 3.1377410114395854, + "itl_p50_ms": 3.1896252185106277, + "itl_p95_ms": 3.6559998989105225, + "round_gap_p50_ms": 3.192916978150606, + "round_gap_p95_ms": 3.6559998989105225, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.1377410114395854, + "itl_p50_ms": 3.1896252185106277, + "itl_p95_ms": 3.6559998989105225, + "round_gap_p50_ms": 3.192916978150606, + "round_gap_p95_ms": 3.6559998989105225, + "round_gap_mean_ms": 3.213349228582708, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2537.772350225976 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 318.002677355984, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8228364591486752, + "end_to_end_tps": 311.1189315369706, + "timing_valid": true, + "per_stream_tps": [ + 318.0 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.020887834019958973, + "decode_tps": 318.002677355984, + "prefill_tps": 2537.3621769187203, + "itl_mean_ms": 3.1446276123032852, + "itl_p50_ms": 3.2035838812589645, + "itl_p95_ms": 3.662958275526762, + "round_gap_p50_ms": 3.217375371605158, + "round_gap_p95_ms": 3.662958275526762, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.1446276123032852, + "itl_p50_ms": 3.2035838812589645, + "itl_p95_ms": 3.662958275526762, + "round_gap_p50_ms": 3.217375371605158, + "round_gap_p95_ms": 3.662958275526762, + "round_gap_mean_ms": 3.220401771635894, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2537.3621769187203 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 837.2, + "per_stream_tps": 214.8, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.052, + "decode_tps": 214.838, + "prefill_tps": 1062.568, + "itl_mean_ms": 4.655, + "itl_p50_ms": 4.601, + "itl_p95_ms": 5.014, + "round_gap_p50_ms": 4.609, + "round_gap_p95_ms": 5.014, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2563.923, + "itl_mean_ms": 4.656, + "itl_p50_ms": 4.6, + "itl_p95_ms": 5.012, + "round_gap_p50_ms": 4.611, + "round_gap_p95_ms": 5.012, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 825.9672050316059, + "makespan_s": 1.2397586656734347, + "steady_tps": 881.0, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 837.218292942188, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2400512918829918, + "end_to_end_tps": 825.7722940194494, + "timing_valid": true, + "steady_tps": 880.7265681044099, + "per_stream_tps": [ + 212.9, + 216.6, + 220.4, + 209.3 + ], + "ttft_s": [ + 0.04, + 0.06, + 0.08, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.04167120810598135, + "decode_tps": 212.88990646245375, + "prefill_tps": 1271.8613740500734, + "itl_mean_ms": 4.6972635603856805, + "itl_p50_ms": 4.605375230312347, + "itl_p95_ms": 5.023249890655279, + "round_gap_p50_ms": 4.614833276718855, + "round_gap_p95_ms": 5.023249890655279, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06211370835080743, + "decode_tps": 216.59588069071452, + "prefill_tps": 853.2738006989571, + "itl_mean_ms": 4.616892975115893, + "itl_p50_ms": 4.602041095495224, + "itl_p95_ms": 5.0109997391700745, + "round_gap_p50_ms": 4.615542013198137, + "round_gap_p95_ms": 5.0109997391700745, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08256666688248515, + "decode_tps": 220.4280864639685, + "prefill_tps": 641.9055292062768, + "itl_mean_ms": 4.53662696093613, + "itl_p50_ms": 4.606250207871199, + "itl_p95_ms": 5.008374806493521, + "round_gap_p50_ms": 4.615291021764278, + "round_gap_p95_ms": 5.008374806493521, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021205416880548, + "decode_tps": 209.3101997531404, + "prefill_tps": 2499.3613800923467, + "itl_mean_ms": 4.777598039557536, + "itl_p50_ms": 4.609167110174894, + "itl_p95_ms": 5.0171660259366035, + "round_gap_p50_ms": 4.612250253558159, + "round_gap_p95_ms": 5.0171660259366035, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.6570953839988105, + "itl_p50_ms": 4.605500027537346, + "itl_p95_ms": 5.012041889131069, + "round_gap_p50_ms": 4.615542013198137, + "round_gap_p95_ms": 5.01208333298564, + "round_gap_mean_ms": 4.750237291678786, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2563.922956544264 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 837.1872267608164, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2397586656734347, + "end_to_end_tps": 825.9672050316059, + "timing_valid": true, + "steady_tps": 880.9701382466629, + "per_stream_tps": [ + 209.4, + 216.6, + 212.9, + 220.5 + ], + "ttft_s": [ + 0.02, + 0.06, + 0.04, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02120154071599245, + "decode_tps": 209.3589684030923, + "prefill_tps": 2499.8183249966255, + "itl_mean_ms": 4.776485132820466, + "itl_p50_ms": 4.591832868754864, + "itl_p95_ms": 5.02233300358057, + "round_gap_p50_ms": 4.60291700437665, + "round_gap_p95_ms": 5.02233300358057, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06219354085624218, + "decode_tps": 216.58944164461542, + "prefill_tps": 852.1785264245901, + "itl_mean_ms": 4.617030231976042, + "itl_p50_ms": 4.60437498986721, + "itl_p95_ms": 5.013250280171633, + "round_gap_p50_ms": 4.616541787981987, + "round_gap_p95_ms": 5.013250280171633, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.041594875045120716, + "decode_tps": 212.93862398531954, + "prefill_tps": 1274.1954373587464, + "itl_mean_ms": 4.6961888890056045, + "itl_p50_ms": 4.592500161379576, + "itl_p95_ms": 5.014082882553339, + "round_gap_p50_ms": 4.604334011673927, + "round_gap_p95_ms": 5.014082882553339, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08276070794090629, + "decode_tps": 220.45331250664685, + "prefill_tps": 640.4005151556177, + "itl_mean_ms": 4.536107843559162, + "itl_p50_ms": 4.593125078827143, + "itl_p95_ms": 5.014415830373764, + "round_gap_p50_ms": 4.605874884873629, + "round_gap_p95_ms": 5.014415830373764, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.656453024340319, + "itl_p50_ms": 4.593125078827143, + "itl_p95_ms": 5.014415830373764, + "round_gap_p50_ms": 4.607209004461765, + "round_gap_p95_ms": 5.014583934098482, + "round_gap_mean_ms": 4.749582084827125, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2561.6020606224706 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 837.8425556187452, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2388595417141914, + "end_to_end_tps": 826.5666651629502, + "timing_valid": true, + "steady_tps": 881.5735503338805, + "per_stream_tps": [ + 216.8, + 220.6, + 213.1, + 209.5 + ], + "ttft_s": [ + 0.06, + 0.08, + 0.04, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.06213716696947813, + "decode_tps": 216.76721323442604, + "prefill_tps": 852.9516645976744, + "itl_mean_ms": 4.613243788480759, + "itl_p50_ms": 4.599666688591242, + "itl_p95_ms": 5.0158328376710415, + "round_gap_p50_ms": 4.602415952831507, + "round_gap_p95_ms": 5.0158328376710415, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08258154103532434, + "decode_tps": 220.59970901608787, + "prefill_tps": 641.7899125584154, + "itl_mean_ms": 4.53309754786246, + "itl_p50_ms": 4.597458057105541, + "itl_p95_ms": 5.002167075872421, + "round_gap_p50_ms": 4.60112513974309, + "round_gap_p95_ms": 5.002167075872421, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04157254192978144, + "decode_tps": 213.08607547852833, + "prefill_tps": 1274.8799457468883, + "itl_mean_ms": 4.692939216015386, + "itl_p50_ms": 4.603083245456219, + "itl_p95_ms": 5.015292204916477, + "round_gap_p50_ms": 4.611040931195021, + "round_gap_p95_ms": 5.015292204916477, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021181500051170588, + "decode_tps": 209.50265701911252, + "prefill_tps": 2502.1835031495316, + "itl_mean_ms": 4.7732091527066975, + "itl_p50_ms": 4.600167274475098, + "itl_p95_ms": 5.0107501447200775, + "round_gap_p50_ms": 4.607457667589188, + "round_gap_p95_ms": 5.0107501447200775, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.653122426266325, + "itl_p50_ms": 4.599666688591242, + "itl_p95_ms": 5.008665844798088, + "round_gap_p50_ms": 4.611040931195021, + "round_gap_p95_ms": 5.0093331374228, + "round_gap_mean_ms": 4.746184874791652, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2564.9052585504696 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 332.7, + "per_stream_tps": 332.7, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 332.673, + "prefill_tps": 2548.923, + "itl_mean_ms": 3.006, + "itl_p50_ms": 3.224, + "itl_p95_ms": 3.675, + "round_gap_p50_ms": 3.281, + "round_gap_p95_ms": 3.676, + "tokens_per_chunk": 1.103 + }, + "pooled": { + "aggregate_prefill_tps": 2548.923, + "itl_mean_ms": 3.006, + "itl_p50_ms": 3.224, + "itl_p95_ms": 3.675, + "round_gap_p50_ms": 3.281, + "round_gap_p95_ms": 3.676, + "tokens_per_chunk": 1.103 + }, + "timing_valid": true, + "end_to_end_tps": 324.97628384693655, + "makespan_s": 0.787749791983515, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 332.67316892728405, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.787749791983515, + "end_to_end_tps": 324.97628384693655, + "timing_valid": true, + "per_stream_tps": [ + 332.7 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02118049981072545, + "decode_tps": 332.67316892728405, + "prefill_tps": 2549.514906756606, + "itl_mean_ms": 3.005953270065434, + "itl_p50_ms": 3.224041312932968, + "itl_p95_ms": 3.6754589527845383, + "round_gap_p50_ms": 3.2811672426760197, + "round_gap_p95_ms": 3.6757499910891056, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.005953270065434, + "itl_p50_ms": 3.224041312932968, + "itl_p95_ms": 3.6754589527845383, + "round_gap_p50_ms": 3.2811672426760197, + "round_gap_p95_ms": 3.6757499910891056, + "round_gap_mean_ms": 3.3182601033189854, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2549.514906756606 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 332.1166396054653, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7890538331121206, + "end_to_end_tps": 324.43920713280875, + "timing_valid": true, + "per_stream_tps": [ + 332.1 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02120170835405588, + "decode_tps": 332.1166396054653, + "prefill_tps": 2546.9645699408848, + "itl_mean_ms": 3.0109903592543277, + "itl_p50_ms": 3.229707945138216, + "itl_p95_ms": 3.6845416761934757, + "round_gap_p50_ms": 3.294500056654215, + "round_gap_p95_ms": 3.688374999910593, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.0109903592543277, + "itl_p50_ms": 3.229707945138216, + "itl_p95_ms": 3.6845416761934757, + "round_gap_p50_ms": 3.294500056654215, + "round_gap_p95_ms": 3.688374999910593, + "round_gap_mean_ms": 3.323820526449582, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2546.9645699408848 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 333.51782137092425, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7858209158293903, + "end_to_end_tps": 325.77397068873665, + "timing_valid": true, + "per_stream_tps": [ + 333.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02118541579693556, + "decode_tps": 333.51782137092425, + "prefill_tps": 2548.9233025962617, + "itl_mean_ms": 2.99834052612092, + "itl_p50_ms": 3.218583296984434, + "itl_p95_ms": 3.669791854918003, + "round_gap_p50_ms": 3.265166189521551, + "round_gap_p95_ms": 3.6720000207424164, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 2.99834052612092, + "itl_p50_ms": 3.218583296984434, + "itl_p95_ms": 3.669791854918003, + "round_gap_p50_ms": 3.265166189521551, + "round_gap_p95_ms": 3.6720000207424164, + "round_gap_mean_ms": 3.3098564249386775, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2548.9233025962617 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 857.9, + "per_stream_tps": 220.3, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.053, + "decode_tps": 220.3, + "prefill_tps": 1068.046, + "itl_mean_ms": 4.54, + "itl_p50_ms": 4.627, + "itl_p95_ms": 5.033, + "round_gap_p50_ms": 4.664, + "round_gap_p95_ms": 5.039, + "tokens_per_chunk": 1.08 + }, + "pooled": { + "aggregate_prefill_tps": 2577.867, + "itl_mean_ms": 4.54, + "itl_p50_ms": 4.624, + "itl_p95_ms": 5.029, + "round_gap_p50_ms": 4.664, + "round_gap_p95_ms": 5.033, + "tokens_per_chunk": 1.08 + }, + "timing_valid": true, + "end_to_end_tps": 845.6963430320134, + "makespan_s": 1.210836499929428, + "steady_tps": 904.4, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 857.6292666665865, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2112241671420634, + "end_to_end_tps": 845.4256675014774, + "timing_valid": true, + "steady_tps": 904.0460851317916, + "per_stream_tps": [ + 218.2, + 222.2, + 226.3, + 214.4 + ], + "ttft_s": [ + 0.04, + 0.06, + 0.08, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.042173834051936865, + "decode_tps": 218.22235854651265, + "prefill_tps": 1280.4147693448804, + "itl_mean_ms": 4.582481862356265, + "itl_p50_ms": 4.6192919835448265, + "itl_p95_ms": 5.038416013121605, + "round_gap_p50_ms": 4.672124981880188, + "round_gap_p95_ms": 5.040541291236877, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06288433400914073, + "decode_tps": 222.1714229049959, + "prefill_tps": 858.7194386466855, + "itl_mean_ms": 4.5010289213821, + "itl_p50_ms": 4.629125352948904, + "itl_p95_ms": 5.042333155870438, + "round_gap_p50_ms": 4.666750319302082, + "round_gap_p95_ms": 5.045500118285418, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08366666687652469, + "decode_tps": 226.25796928066754, + "prefill_tps": 645.4183250743481, + "itl_mean_ms": 4.41973382497535, + "itl_p50_ms": 4.627750255167484, + "itl_p95_ms": 5.037125200033188, + "round_gap_p50_ms": 4.662957973778248, + "round_gap_p95_ms": 5.038124974817038, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.02144520776346326, + "decode_tps": 214.41120032866718, + "prefill_tps": 2518.045084739219, + "itl_mean_ms": 4.663935458908479, + "itl_p50_ms": 4.627666901797056, + "itl_p95_ms": 5.047291982918978, + "round_gap_p50_ms": 4.66120894998312, + "round_gap_p95_ms": 5.054167006164789, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.5417950169055485, + "itl_p50_ms": 4.627542104572058, + "itl_p95_ms": 5.038416013121605, + "round_gap_p50_ms": 4.66966675594449, + "round_gap_p95_ms": 5.041208118200302, + "round_gap_mean_ms": 4.907448005554724, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2579.468934585024 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 857.9225871056058, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.210836499929428, + "end_to_end_tps": 845.6963430320134, + "timing_valid": true, + "steady_tps": 904.3955647231896, + "per_stream_tps": [ + 222.3, + 214.5, + 226.3, + 218.3 + ], + "ttft_s": [ + 0.06, + 0.02, + 0.08, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.06300683319568634, + "decode_tps": 222.25744113706236, + "prefill_tps": 857.0498985766677, + "itl_mean_ms": 4.499286929985471, + "itl_p50_ms": 4.6308329328894615, + "itl_p95_ms": 5.032957997173071, + "round_gap_p50_ms": 4.663874860852957, + "round_gap_p95_ms": 5.044875200837851, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021404666360467672, + "decode_tps": 214.4874345274434, + "prefill_tps": 2522.8143756416, + "itl_mean_ms": 4.662277779596693, + "itl_p50_ms": 4.629999864846468, + "itl_p95_ms": 5.029458086937666, + "round_gap_p50_ms": 4.661249928176403, + "round_gap_p95_ms": 5.030333995819092, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08366304216906428, + "decode_tps": 226.34367429604364, + "prefill_tps": 645.4462878707911, + "itl_mean_ms": 4.418060293092447, + "itl_p50_ms": 4.618499893695116, + "itl_p95_ms": 5.021874792873859, + "round_gap_p50_ms": 4.672250244766474, + "round_gap_p95_ms": 5.039040930569172, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04227425018325448, + "decode_tps": 218.30240987476265, + "prefill_tps": 1277.3733363907252, + "itl_mean_ms": 4.580801469730396, + "itl_p50_ms": 4.622458014637232, + "itl_p95_ms": 5.03287510946393, + "round_gap_p50_ms": 4.663208033889532, + "round_gap_p95_ms": 5.03425020724535, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.540106618101252, + "itl_p50_ms": 4.624207969754934, + "itl_p95_ms": 5.029458086937666, + "round_gap_p50_ms": 4.663874860852957, + "round_gap_p95_ms": 5.032957997173071, + "round_gap_mean_ms": 4.905623676338217, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2577.8668200705283 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 858.4259581753395, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.210179541260004, + "end_to_end_tps": 846.1554381706377, + "timing_valid": true, + "steady_tps": 904.9365660766687, + "per_stream_tps": [ + 218.4, + 222.4, + 226.5, + 214.6 + ], + "ttft_s": [ + 0.04, + 0.06, + 0.08, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.0422492497600615, + "decode_tps": 218.42901272972236, + "prefill_tps": 1278.1292048183673, + "itl_mean_ms": 4.578146407855492, + "itl_p50_ms": 4.628082737326622, + "itl_p95_ms": 5.020291078835726, + "round_gap_p50_ms": 4.6661668457090855, + "round_gap_p95_ms": 5.039167124778032, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06298808380961418, + "decode_tps": 222.3799635887344, + "prefill_tps": 857.3050128532044, + "itl_mean_ms": 4.4968080031229, + "itl_p50_ms": 4.621000029146671, + "itl_p95_ms": 5.033832974731922, + "round_gap_p50_ms": 4.6539162285625935, + "round_gap_p95_ms": 5.04204211756587, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08368041599169374, + "decode_tps": 226.48035708669286, + "prefill_tps": 645.3122795823593, + "itl_mean_ms": 4.415393956736023, + "itl_p50_ms": 4.622624721378088, + "itl_p95_ms": 5.025958176702261, + "round_gap_p50_ms": 4.6584997326135635, + "round_gap_p95_ms": 5.026041064411402, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021520750131458044, + "decode_tps": 214.61448188984392, + "prefill_tps": 2509.206215868158, + "itl_mean_ms": 4.659517806972943, + "itl_p50_ms": 4.626083187758923, + "itl_p95_ms": 5.022000055760145, + "round_gap_p50_ms": 4.664500243961811, + "round_gap_p95_ms": 5.027707666158676, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.537466543671839, + "itl_p50_ms": 4.624208901077509, + "itl_p95_ms": 5.025958176702261, + "round_gap_p50_ms": 4.663832951337099, + "round_gap_p95_ms": 5.032999906688929, + "round_gap_mean_ms": 4.9027710535437246, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2577.456720240308 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + } + ] + } + ] + }, + "text-4-2-new.decode.json": [ + { + "label": "text-4-2-new", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 317.8355443591904, + "decode_tps_all": [ + 318.86, + 317.84, + 317.51 + ], + "ttft_s_median": 0.02093658410012722, + "prefill_tps_median": 2531.454, + "itl_mean_ms_median": 3.146, + "itl_p50_ms_median": 3.196, + "itl_p95_ms_median": 3.665, + "round_gap_p50_ms_median": 3.213, + "round_gap_p95_ms_median": 3.665, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "sample": "Here is a complete Python implementation of a Red-Black Tree with insertion, deletion, searching, rotations, and extensive unit tests. I'll provide detailed exp" + }, + { + "label": "text-4-2-new", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 333.4991928050717, + "decode_tps_all": [ + 333.5, + 334.01, + 333.12 + ], + "ttft_s_median": 0.021197374910116196, + "prefill_tps_median": 2547.485, + "itl_mean_ms_median": 2.999, + "itl_p50_ms_median": 3.213, + "itl_p95_ms_median": 3.674, + "round_gap_p50_ms_median": 3.275, + "round_gap_p95_ms_median": 3.675, + "tokens_per_chunk_median": 1.103, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "sample": "We need to write a detailed 2000-word tutorial explaining how matrix multiplication runs on a GPU, covering memory layout, tiling, shared memory, synchro" + } + ], + "text-4-3-old.concurrent.json": { + "label": "text-4-3-old", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 309.3, + "per_stream_tps": 309.3, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 309.266, + "prefill_tps": 2521.403, + "itl_mean_ms": 3.233, + "itl_p50_ms": 3.294, + "itl_p95_ms": 3.748, + "round_gap_p50_ms": 3.304, + "round_gap_p95_ms": 3.748, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2521.403, + "itl_mean_ms": 3.233, + "itl_p50_ms": 3.294, + "itl_p95_ms": 3.748, + "round_gap_p50_ms": 3.304, + "round_gap_p95_ms": 3.748, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 302.75900973349377, + "makespan_s": 0.8455570000223815, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 309.38753074380236, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8453007908537984, + "end_to_end_tps": 302.85077545169037, + "timing_valid": true, + "per_stream_tps": [ + 309.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021038499660789967, + "decode_tps": 309.38753074380236, + "prefill_tps": 2519.1910475810955, + "itl_mean_ms": 3.232192317498665, + "itl_p50_ms": 3.2839998602867126, + "itl_p95_ms": 3.747750073671341, + "round_gap_p50_ms": 3.2929996959865093, + "round_gap_p95_ms": 3.747750073671341, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.232192317498665, + "itl_p50_ms": 3.2839998602867126, + "itl_p95_ms": 3.747750073671341, + "round_gap_p50_ms": 3.2929996959865093, + "round_gap_p95_ms": 3.747750073671341, + "round_gap_mean_ms": 3.310076469727549, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2519.1910475810955 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 308.98231262341164, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.846363291144371, + "end_to_end_tps": 302.4705852422562, + "timing_valid": true, + "per_stream_tps": [ + 309.0 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021020041313022375, + "decode_tps": 308.98231262341164, + "prefill_tps": 2521.4032270795465, + "itl_mean_ms": 3.236431210283555, + "itl_p50_ms": 3.2941666431725025, + "itl_p95_ms": 3.7594158202409744, + "round_gap_p50_ms": 3.308167215436697, + "round_gap_p95_ms": 3.7594158202409744, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.236431210283555, + "itl_p50_ms": 3.2941666431725025, + "itl_p95_ms": 3.7594158202409744, + "round_gap_p50_ms": 3.308167215436697, + "round_gap_p95_ms": 3.7594158202409744, + "round_gap_mean_ms": 3.3144175045072553, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2521.4032270795465 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 309.2663944831718, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8455570000223815, + "end_to_end_tps": 302.75900973349377, + "timing_valid": true, + "per_stream_tps": [ + 309.3 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02096254238858819, + "decode_tps": 309.2663944831718, + "prefill_tps": 2528.31927623687, + "itl_mean_ms": 3.23345833184088, + "itl_p50_ms": 3.2971249893307686, + "itl_p95_ms": 3.7459167651832104, + "round_gap_p50_ms": 3.303749952465296, + "round_gap_p95_ms": 3.7459167651832104, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.23345833184088, + "itl_p50_ms": 3.2971249893307686, + "itl_p95_ms": 3.7459167651832104, + "round_gap_p50_ms": 3.303749952465296, + "round_gap_p95_ms": 3.7459167651832104, + "round_gap_mean_ms": 3.3113729904394553, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2528.31927623687 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 682.6, + "per_stream_tps": 173.8, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.053, + "decode_tps": 173.761, + "prefill_tps": 1031.381, + "itl_mean_ms": 5.755, + "itl_p50_ms": 6.205, + "itl_p95_ms": 7.007, + "round_gap_p50_ms": 6.227, + "round_gap_p95_ms": 7.007, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2505.703, + "itl_mean_ms": 5.736, + "itl_p50_ms": 6.204, + "itl_p95_ms": 7.008, + "round_gap_p50_ms": 6.223, + "round_gap_p95_ms": 7.011, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 674.8388358518314, + "makespan_s": 1.517399333883077, + "steady_tps": 711.7, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 693.2723332337463, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.4926918330602348, + "end_to_end_tps": 686.0089787593006, + "timing_valid": true, + "steady_tps": 723.0582063101217, + "per_stream_tps": [ + 180.9, + 173.4, + 175.8, + 178.4 + ], + "ttft_s": [ + 0.08, + 0.02, + 0.04, + 0.06 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.0828781658783555, + "decode_tps": 180.90283283362396, + "prefill_tps": 639.4929163585834, + "itl_mean_ms": 5.527829411713516, + "itl_p50_ms": 5.067375022917986, + "itl_p95_ms": 6.989667192101479, + "round_gap_p50_ms": 5.099749658256769, + "round_gap_p95_ms": 6.989667192101479, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021158374845981598, + "decode_tps": 173.38157165035227, + "prefill_tps": 2504.918283460025, + "itl_mean_ms": 5.7676256506466395, + "itl_p50_ms": 5.095415748655796, + "itl_p95_ms": 7.011957932263613, + "round_gap_p50_ms": 5.1423329859972, + "round_gap_p95_ms": 7.011957932263613, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.0416510826908052, + "decode_tps": 175.83274316941646, + "prefill_tps": 1272.4759256186194, + "itl_mean_ms": 5.687222879964347, + "itl_p50_ms": 5.11637469753623, + "itl_p95_ms": 7.015874609351158, + "round_gap_p50_ms": 5.143875256180763, + "round_gap_p95_ms": 7.015874609351158, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06235062517225742, + "decode_tps": 178.36166413161467, + "prefill_tps": 850.0315731169616, + "itl_mean_ms": 5.606585949221078, + "itl_p50_ms": 5.119333975017071, + "itl_p95_ms": 7.001708261668682, + "round_gap_p50_ms": 5.135875195264816, + "round_gap_p95_ms": 7.001708261668682, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.647315972886395, + "itl_p50_ms": 5.095415748655796, + "itl_p95_ms": 7.008375134319067, + "round_gap_p50_ms": 5.1423329859972, + "round_gap_p95_ms": 7.01058330014348, + "round_gap_mean_ms": 5.760262292344123, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2556.8674455896603 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 682.6332198696525, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.517399333883077, + "end_to_end_tps": 674.8388358518314, + "timing_valid": true, + "steady_tps": 711.6786746471587, + "per_stream_tps": [ + 178.1, + 173.1, + 175.6, + 170.7 + ], + "ttft_s": [ + 0.08, + 0.04, + 0.06, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.08488504216074944, + "decode_tps": 178.08551264421712, + "prefill_tps": 624.3738431516857, + "itl_mean_ms": 5.615279902064917, + "itl_p50_ms": 6.202958058565855, + "itl_p95_ms": 6.707292050123215, + "round_gap_p50_ms": 6.219707895070314, + "round_gap_p95_ms": 6.707292050123215, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04370304103940725, + "decode_tps": 173.13032611964786, + "prefill_tps": 1212.7302526203985, + "itl_mean_ms": 5.775995589062279, + "itl_p50_ms": 6.2017920427024364, + "itl_p95_ms": 6.709542125463486, + "round_gap_p50_ms": 6.209040991961956, + "round_gap_p95_ms": 6.709542125463486, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06427412526682019, + "decode_tps": 175.5902018412655, + "prefill_tps": 824.5930968330088, + "itl_mean_ms": 5.695078595011842, + "itl_p50_ms": 6.207332946360111, + "itl_p95_ms": 6.70829089358449, + "round_gap_p50_ms": 6.23737508431077, + "round_gap_p95_ms": 6.70829089358449, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.02263020910322666, + "decode_tps": 170.69351353921118, + "prefill_tps": 2342.002221819645, + "itl_mean_ms": 5.8584534307467, + "itl_p50_ms": 6.209291983395815, + "itl_p95_ms": 6.716167088598013, + "round_gap_p50_ms": 6.234750151634216, + "round_gap_p95_ms": 6.716167088598013, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.736201879221435, + "itl_p50_ms": 6.20441697537899, + "itl_p95_ms": 6.7083751782774925, + "round_gap_p50_ms": 6.222500000149012, + "round_gap_p95_ms": 6.7092920653522015, + "round_gap_mean_ms": 5.850925916805863, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2495.65417480838 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 668.0776347728244, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5490623330697417, + "end_to_end_tps": 661.0450581228467, + "timing_valid": true, + "steady_tps": 696.0018522965812, + "per_stream_tps": [ + 174.1, + 171.8, + 169.4, + 167.1 + ], + "ttft_s": [ + 0.08, + 0.06, + 0.04, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.08452612487599254, + "decode_tps": 174.1410945981182, + "prefill_tps": 627.0250774864669, + "itl_mean_ms": 5.742469933979652, + "itl_p50_ms": 6.392166949808598, + "itl_p95_ms": 7.038250099867582, + "round_gap_p50_ms": 6.409292109310627, + "round_gap_p95_ms": 7.038250099867582, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06388687482103705, + "decode_tps": 171.7843767986256, + "prefill_tps": 829.5913698778679, + "itl_mean_ms": 5.82125114423095, + "itl_p50_ms": 6.411832757294178, + "itl_p95_ms": 7.052749861031771, + "round_gap_p50_ms": 6.4193750731647015, + "round_gap_p95_ms": 7.052749861031771, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.043389542028307915, + "decode_tps": 169.44071382848637, + "prefill_tps": 1221.492496173896, + "itl_mean_ms": 5.901769282040643, + "itl_p50_ms": 6.413042079657316, + "itl_p95_ms": 7.070166990160942, + "round_gap_p50_ms": 6.438165903091431, + "round_gap_p95_ms": 7.070166990160942, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.022168166004121304, + "decode_tps": 167.07373140230848, + "prefill_tps": 2390.8157305456266, + "itl_mean_ms": 5.985381373879957, + "itl_p50_ms": 6.407833192497492, + "itl_p95_ms": 7.053666282445192, + "round_gap_p50_ms": 6.410208065062761, + "round_gap_p95_ms": 7.053666282445192, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.8627179335328, + "itl_p50_ms": 6.407833192497492, + "itl_p95_ms": 7.049958687275648, + "round_gap_p50_ms": 6.4170826226472855, + "round_gap_p95_ms": 7.052749861031771, + "round_gap_mean_ms": 5.979972292203456, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2505.7028379140365 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 322.1, + "per_stream_tps": 322.1, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 322.086, + "prefill_tps": 2530.924, + "itl_mean_ms": 3.105, + "itl_p50_ms": 3.32, + "itl_p95_ms": 3.768, + "round_gap_p50_ms": 3.369, + "round_gap_p95_ms": 3.774, + "tokens_per_chunk": 1.103 + }, + "pooled": { + "aggregate_prefill_tps": 2530.924, + "itl_mean_ms": 3.105, + "itl_p50_ms": 3.32, + "itl_p95_ms": 3.768, + "round_gap_p50_ms": 3.369, + "round_gap_p95_ms": 3.774, + "tokens_per_chunk": 1.103 + }, + "timing_valid": true, + "end_to_end_tps": 314.83719447360625, + "makespan_s": 0.8131186673417687, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 322.0860565549538, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8131186673417687, + "end_to_end_tps": 314.83719447360625, + "timing_valid": true, + "per_stream_tps": [ + 322.1 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021354292053729296, + "decode_tps": 322.0860565549538, + "prefill_tps": 2528.7656394382543, + "itl_mean_ms": 3.104760295108837, + "itl_p50_ms": 3.323208075016737, + "itl_p95_ms": 3.764667082577944, + "round_gap_p50_ms": 3.353250212967396, + "round_gap_p95_ms": 3.7722918204963207, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.104760295108837, + "itl_p50_ms": 3.323208075016737, + "itl_p95_ms": 3.764667082577944, + "round_gap_p50_ms": 3.353250212967396, + "round_gap_p95_ms": 3.7722918204963207, + "round_gap_mean_ms": 3.427332793301963, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2528.7656394382543 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 322.1968181152365, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8128247922286391, + "end_to_end_tps": 314.9510232064746, + "timing_valid": true, + "per_stream_tps": [ + 322.2 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02133179223164916, + "decode_tps": 322.1968181152365, + "prefill_tps": 2531.43286853705, + "itl_mean_ms": 3.1036929720464874, + "itl_p50_ms": 3.3198753371834755, + "itl_p95_ms": 3.7679998204112053, + "round_gap_p50_ms": 3.369082696735859, + "round_gap_p95_ms": 3.773542121052742, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.1036929720464874, + "itl_p50_ms": 3.3198753371834755, + "itl_p95_ms": 3.7679998204112053, + "round_gap_p50_ms": 3.369082696735859, + "round_gap_p95_ms": 3.773542121052742, + "round_gap_mean_ms": 3.4261545795318367, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2531.43286853705 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 321.9407006996018, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8134599579498172, + "end_to_end_tps": 314.7051031807429, + "timing_valid": true, + "per_stream_tps": [ + 321.9 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021336082834750414, + "decode_tps": 321.9407006996018, + "prefill_tps": 2530.92380725338, + "itl_mean_ms": 3.106162090804063, + "itl_p50_ms": 3.3197500742971897, + "itl_p95_ms": 3.7722080014646053, + "round_gap_p50_ms": 3.3699581399559975, + "round_gap_p95_ms": 3.773957956582308, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.106162090804063, + "itl_p50_ms": 3.3197500742971897, + "itl_p95_ms": 3.7722080014646053, + "round_gap_p50_ms": 3.3699581399559975, + "round_gap_p95_ms": 3.773957956582308, + "round_gap_mean_ms": 3.4288802301083807, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2530.92380725338 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 731.1, + "per_stream_tps": 184.9, + "ttft_s_max": 0.09, + "request": { + "ttft_s": 0.054, + "decode_tps": 184.931, + "prefill_tps": 1029.926, + "itl_mean_ms": 5.408, + "itl_p50_ms": 4.974, + "itl_p95_ms": 7.072, + "round_gap_p50_ms": 5.023, + "round_gap_p95_ms": 7.096, + "tokens_per_chunk": 1.08 + }, + "pooled": { + "aggregate_prefill_tps": 2498.83, + "itl_mean_ms": 5.346, + "itl_p50_ms": 4.97, + "itl_p95_ms": 7.05, + "round_gap_p50_ms": 5.024, + "round_gap_p95_ms": 7.094, + "tokens_per_chunk": 1.08 + }, + "timing_valid": true, + "end_to_end_tps": 721.745496784291, + "makespan_s": 1.4187826658599079, + "steady_tps": 765.2, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 657.5379733510061, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5732247079722583, + "end_to_end_tps": 650.8923962425188, + "timing_valid": true, + "steady_tps": 684.7165499164536, + "per_stream_tps": [ + 169.0, + 166.7, + 171.3, + 164.4 + ], + "ttft_s": [ + 0.06, + 0.04, + 0.08, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.06345533393323421, + "decode_tps": 168.98864062399034, + "prefill_tps": 850.9922909997947, + "itl_mean_ms": 5.91755751337783, + "itl_p50_ms": 6.434833165258169, + "itl_p95_ms": 7.050416897982359, + "round_gap_p50_ms": 6.479917094111443, + "round_gap_p95_ms": 7.0538329891860485, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04246604209765792, + "decode_tps": 166.68070197881931, + "prefill_tps": 1271.604259135282, + "itl_mean_ms": 5.9994947713087585, + "itl_p50_ms": 6.454874761402607, + "itl_p95_ms": 7.094374857842922, + "round_gap_p50_ms": 6.484375335276127, + "round_gap_p95_ms": 7.108666934072971, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08433179184794426, + "decode_tps": 171.31403103667543, + "prefill_tps": 640.3279097563293, + "itl_mean_ms": 5.837233494236773, + "itl_p50_ms": 6.454042159020901, + "itl_p95_ms": 7.064125034958124, + "round_gap_p50_ms": 6.492167245596647, + "round_gap_p95_ms": 7.086958270519972, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.02165025006979704, + "decode_tps": 164.43619628562223, + "prefill_tps": 2494.197518546548, + "itl_mean_ms": 6.081386109558391, + "itl_p50_ms": 6.457250099629164, + "itl_p95_ms": 7.102624978870153, + "round_gap_p50_ms": 6.474332883954048, + "round_gap_p95_ms": 7.104584015905857, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.958917972120438, + "itl_p50_ms": 6.454042159020901, + "itl_p95_ms": 7.050416897982359, + "round_gap_p50_ms": 6.484499666839838, + "round_gap_p95_ms": 7.094374857842922, + "round_gap_mean_ms": 6.4386613681809814, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2559.3351974847587 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 731.1139639675918, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.4187826658599079, + "end_to_end_tps": 721.745496784291, + "timing_valid": true, + "steady_tps": 765.1674598272779, + "per_stream_tps": [ + 191.5, + 188.5, + 182.8, + 185.7 + ], + "ttft_s": [ + 0.09, + 0.07, + 0.02, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.08636387484148145, + "decode_tps": 191.49479009997836, + "prefill_tps": 625.2614313463301, + "itl_mean_ms": 5.222074185297186, + "itl_p50_ms": 4.971332848072052, + "itl_p95_ms": 6.960625294595957, + "round_gap_p50_ms": 5.00691682100296, + "round_gap_p95_ms": 7.000165991485119, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06541170831769705, + "decode_tps": 188.54227365856931, + "prefill_tps": 825.5402800019882, + "itl_mean_ms": 5.303850328074951, + "itl_p50_ms": 4.969374742358923, + "itl_p95_ms": 7.0107909850776196, + "round_gap_p50_ms": 5.0184582360088825, + "round_gap_p95_ms": 7.027625106275082, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.022814041003584862, + "decode_tps": 182.78812085430368, + "prefill_tps": 2366.9633973005816, + "itl_mean_ms": 5.4708150361536765, + "itl_p50_ms": 4.973332863301039, + "itl_p95_ms": 7.045208010822535, + "round_gap_p50_ms": 5.0235409289598465, + "round_gap_p95_ms": 7.045582868158817, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04467016598209739, + "decode_tps": 185.67486267935436, + "prefill_tps": 1208.8605182627207, + "itl_mean_ms": 5.3857586620485085, + "itl_p50_ms": 4.969541914761066, + "itl_p95_ms": 7.021917030215263, + "round_gap_p50_ms": 5.022875033318996, + "round_gap_p95_ms": 7.069583982229233, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.34562455289358, + "itl_p50_ms": 4.969541914761066, + "itl_p95_ms": 6.971541792154312, + "round_gap_p50_ms": 5.0235409289598465, + "round_gap_p95_ms": 7.027625106275082, + "round_gap_mean_ms": 5.775992631304504, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2498.829889284282 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 736.4566435983156, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.4077557497657835, + "end_to_end_tps": 727.3989114733638, + "timing_valid": true, + "steady_tps": 771.6789123899893, + "per_stream_tps": [ + 193.1, + 190.1, + 187.1, + 184.2 + ], + "ttft_s": [ + 0.09, + 0.07, + 0.04, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.0866750837303698, + "decode_tps": 193.05555208976284, + "prefill_tps": 623.0164157439298, + "itl_mean_ms": 5.179856208098285, + "itl_p50_ms": 4.961625207215548, + "itl_p95_ms": 7.079957984387875, + "round_gap_p50_ms": 5.027499981224537, + "round_gap_p95_ms": 7.144166622310877, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06571549968793988, + "decode_tps": 190.12179933417977, + "prefill_tps": 821.7239503074202, + "itl_mean_ms": 5.259786113439238, + "itl_p50_ms": 4.9708751030266285, + "itl_p95_ms": 7.117874920368195, + "round_gap_p50_ms": 5.019540898501873, + "round_gap_p95_ms": 7.183624897152185, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04414195800200105, + "decode_tps": 187.10337416852923, + "prefill_tps": 1223.325888660219, + "itl_mean_ms": 5.344639050171656, + "itl_p50_ms": 4.974625073373318, + "itl_p95_ms": 7.125959265977144, + "round_gap_p50_ms": 5.017832852900028, + "round_gap_p95_ms": 7.149290759116411, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.022597874980419874, + "decode_tps": 184.18645522306124, + "prefill_tps": 2389.605219375219, + "itl_mean_ms": 5.429280881642127, + "itl_p50_ms": 4.974500276148319, + "itl_p95_ms": 7.131082937121391, + "round_gap_p50_ms": 5.021417047828436, + "round_gap_p95_ms": 7.137250155210495, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.303390563337826, + "itl_p50_ms": 4.970333073288202, + "itl_p95_ms": 7.087832782417536, + "round_gap_p50_ms": 5.021417047828436, + "round_gap_p95_ms": 7.137250155210495, + "round_gap_mean_ms": 5.730358447674346, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2490.063039496965 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true + } + } + ] + } + ] + } + ] + }, + "text-4-3-old.decode.json": [ + { + "label": "text-4-3-old", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 308.9845434949348, + "decode_tps_all": [ + 309.23, + 308.98, + 308.66 + ], + "ttft_s_median": 0.021053583826869726, + "prefill_tps_median": 2517.386, + "itl_mean_ms_median": 3.236, + "itl_p50_ms_median": 3.286, + "itl_p95_ms_median": 3.757, + "round_gap_p50_ms_median": 3.303, + "round_gap_p95_ms_median": 3.757, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "sample": "Here is a complete Python implementation of a Red-Black Tree with insertion, deletion, searching, rotations, and extensive unit tests. I'll provide detailed exp" + }, + { + "label": "text-4-3-old", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 322.53863627043455, + "decode_tps_all": [ + 322.38, + 322.62, + 322.54 + ], + "ttft_s_median": 0.021351624745875597, + "prefill_tps_median": 2529.082, + "itl_mean_ms_median": 3.1, + "itl_p50_ms_median": 3.309, + "itl_p95_ms_median": 3.76, + "round_gap_p50_ms_median": 3.365, + "round_gap_p95_ms_median": 3.764, + "tokens_per_chunk_median": 1.103, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "sample": "We need to write a detailed 2000-word tutorial explaining how matrix multiplication runs on a GPU, covering memory layout, tiling, shared memory, synchro" + } + ], + "vl-4-0-old.concurrent.json": { + "label": "vl-4-0-old", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 293.9, + "per_stream_tps": 293.9, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 293.873, + "prefill_tps": 2503.296, + "itl_mean_ms": 3.403, + "itl_p50_ms": 3.451, + "itl_p95_ms": 3.949, + "round_gap_p50_ms": 3.464, + "round_gap_p95_ms": 3.949, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2503.296, + "itl_mean_ms": 3.403, + "itl_p50_ms": 3.451, + "itl_p95_ms": 3.949, + "round_gap_p50_ms": 3.464, + "round_gap_p95_ms": 3.949, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 288.0078749127682, + "makespan_s": 0.8888645842671394, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 293.8730495465859, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8888247082941234, + "end_to_end_tps": 288.02079601424214, + "timing_valid": true, + "per_stream_tps": [ + 293.9 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021045792382210493, + "decode_tps": 293.8730495465859, + "prefill_tps": 2518.3181054660427, + "itl_mean_ms": 3.402829900676129, + "itl_p50_ms": 3.4410832449793816, + "itl_p95_ms": 3.9490838535130024, + "round_gap_p50_ms": 3.455167170614004, + "round_gap_p95_ms": 3.9490838535130024, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.402829900676129, + "itl_p50_ms": 3.4410832449793816, + "itl_p95_ms": 3.9490838535130024, + "round_gap_p50_ms": 3.455167170614004, + "round_gap_p95_ms": 3.9490838535130024, + "round_gap_mean_ms": 3.4848258018972404, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2518.3181054660427 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 293.4877091358265, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8900999999605119, + "end_to_end_tps": 287.6081339302967, + "timing_valid": true, + "per_stream_tps": [ + 293.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021172082982957363, + "decode_tps": 293.4877091358265, + "prefill_tps": 2503.2964419543778, + "itl_mean_ms": 3.4072977125498594, + "itl_p50_ms": 3.4511247649788857, + "itl_p95_ms": 3.9510419592261314, + "round_gap_p50_ms": 3.464417066425085, + "round_gap_p95_ms": 3.9510419592261314, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.4072977125498594, + "itl_p50_ms": 3.4511247649788857, + "itl_p95_ms": 3.9510419592261314, + "round_gap_p50_ms": 3.464417066425085, + "round_gap_p95_ms": 3.9510419592261314, + "round_gap_mean_ms": 3.4894012718884104, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2503.2964419543778 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 293.9026298965284, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8888645842671394, + "end_to_end_tps": 288.0078749127682, + "timing_valid": true, + "per_stream_tps": [ + 293.9 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02117895893752575, + "decode_tps": 293.9026298965284, + "prefill_tps": 2502.4837224691164, + "itl_mean_ms": 3.402487416842989, + "itl_p50_ms": 3.460667096078396, + "itl_p95_ms": 3.944333177059889, + "round_gap_p50_ms": 3.4726662561297417, + "round_gap_p95_ms": 3.944333177059889, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.402487416842989, + "itl_p50_ms": 3.460667096078396, + "itl_p95_ms": 3.944333177059889, + "round_gap_p50_ms": 3.4726662561297417, + "round_gap_p95_ms": 3.944333177059889, + "round_gap_mean_ms": 3.484475065441615, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2502.4837224691164 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 661.4, + "per_stream_tps": 168.4, + "ttft_s_max": 0.09, + "request": { + "ttft_s": 0.054, + "decode_tps": 168.338, + "prefill_tps": 1017.467, + "itl_mean_ms": 5.94, + "itl_p50_ms": 6.285, + "itl_p95_ms": 7.135, + "round_gap_p50_ms": 6.344, + "round_gap_p95_ms": 7.135, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2494.515, + "itl_mean_ms": 5.926, + "itl_p50_ms": 6.286, + "itl_p95_ms": 7.131, + "round_gap_p50_ms": 6.347, + "round_gap_p95_ms": 7.132, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 654.6067484021899, + "makespan_s": 1.564297958277166, + "steady_tps": 687.1, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 661.4383672979033, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.564147333148867, + "end_to_end_tps": 654.6697860862838, + "timing_valid": true, + "steady_tps": 687.0685472642173, + "per_stream_tps": [ + 172.3, + 167.6, + 169.9, + 165.4 + ], + "ttft_s": [ + 0.08, + 0.04, + 0.06, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.08346404181793332, + "decode_tps": 172.29597792387472, + "prefill_tps": 635.0039950810562, + "itl_mean_ms": 5.803966012728565, + "itl_p50_ms": 6.1836251989007, + "itl_p95_ms": 7.114042062312365, + "round_gap_p50_ms": 6.219999864697456, + "round_gap_p95_ms": 7.114042062312365, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.042049292009323835, + "decode_tps": 167.61479948777352, + "prefill_tps": 1260.4255022474101, + "itl_mean_ms": 5.9660602945323085, + "itl_p50_ms": 6.229416932910681, + "itl_p95_ms": 7.130333688110113, + "round_gap_p50_ms": 6.239332724362612, + "round_gap_p95_ms": 7.130333688110113, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06268450012430549, + "decode_tps": 169.9233550551976, + "prefill_tps": 845.5040703028533, + "itl_mean_ms": 5.885006211625011, + "itl_p50_ms": 6.209667306393385, + "itl_p95_ms": 7.156709209084511, + "round_gap_p50_ms": 6.245958153158426, + "round_gap_p95_ms": 7.156709209084511, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021421083249151707, + "decode_tps": 165.3668972633712, + "prefill_tps": 2474.1979377769726, + "itl_mean_ms": 6.047159477191813, + "itl_p50_ms": 6.2412079423666, + "itl_p95_ms": 7.148499600589275, + "round_gap_p50_ms": 6.278249900788069, + "round_gap_p95_ms": 7.148499600589275, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.925547999019424, + "itl_p50_ms": 6.209667306393385, + "itl_p95_ms": 7.130333688110113, + "round_gap_p50_ms": 6.262916140258312, + "round_gap_p95_ms": 7.131708785891533, + "round_gap_mean_ms": 6.044058958999813, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2538.8144612378824 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 657.2026266419617, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5755937499925494, + "end_to_end_tps": 649.9137230043228, + "timing_valid": true, + "steady_tps": 682.9356474361905, + "per_stream_tps": [ + 171.2, + 164.3, + 166.6, + 168.9 + ], + "ttft_s": [ + 0.09, + 0.02, + 0.04, + 0.07 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.08598849968984723, + "decode_tps": 171.22968113229572, + "prefill_tps": 616.3614924224312, + "itl_mean_ms": 5.840108989208352, + "itl_p50_ms": 6.421082653105259, + "itl_p95_ms": 7.1258749812841415, + "round_gap_p50_ms": 6.460624746978283, + "round_gap_p95_ms": 7.1258749812841415, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.0231413752771914, + "decode_tps": 164.3384094804152, + "prefill_tps": 2290.2701056076753, + "itl_mean_ms": 6.085004736030219, + "itl_p50_ms": 6.441167090088129, + "itl_p95_ms": 7.116499822586775, + "round_gap_p50_ms": 6.464667152613401, + "round_gap_p95_ms": 7.116499822586775, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04455912485718727, + "decode_tps": 166.59546884595179, + "prefill_tps": 1189.4309004017891, + "itl_mean_ms": 6.002564216945686, + "itl_p50_ms": 6.467041093856096, + "itl_p95_ms": 7.146832998842001, + "round_gap_p50_ms": 6.488332990556955, + "round_gap_p95_ms": 7.146832998842001, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06536312494426966, + "decode_tps": 168.91426594306853, + "prefill_tps": 810.8547448609474, + "itl_mean_ms": 5.920163074544832, + "itl_p50_ms": 6.464541889727116, + "itl_p95_ms": 7.148082833737135, + "round_gap_p50_ms": 6.469834130257368, + "round_gap_p95_ms": 7.148082833737135, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.961960254182272, + "itl_p50_ms": 6.441167090088129, + "itl_p95_ms": 7.130625192075968, + "round_gap_p50_ms": 6.472332868725061, + "round_gap_p95_ms": 7.1316673420369625, + "round_gap_mean_ms": 6.0811994592659175, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2463.6862899914345 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 661.9765842377002, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.564297958277166, + "end_to_end_tps": 654.6067484021899, + "timing_valid": true, + "steady_tps": 687.6910361334991, + "per_stream_tps": [ + 165.5, + 167.8, + 172.4, + 170.1 + ], + "ttft_s": [ + 0.02, + 0.04, + 0.08, + 0.06 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02284037508070469, + "decode_tps": 165.51097011934883, + "prefill_tps": 2320.4522610827808, + "itl_mean_ms": 6.0418955872164055, + "itl_p50_ms": 6.286084186285734, + "itl_p95_ms": 7.132083177566528, + "round_gap_p50_ms": 6.356707774102688, + "round_gap_p95_ms": 7.132083177566528, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.043513457756489515, + "decode_tps": 167.7623301245715, + "prefill_tps": 1218.0139830899943, + "itl_mean_ms": 5.960813725330666, + "itl_p50_ms": 6.284541916102171, + "itl_p95_ms": 7.128917146474123, + "round_gap_p50_ms": 6.373665761202574, + "round_gap_p95_ms": 7.128917146474123, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08485229220241308, + "decode_tps": 172.4417512859152, + "prefill_tps": 624.6148291854013, + "itl_mean_ms": 5.799059639228325, + "itl_p50_ms": 6.273332983255386, + "itl_p95_ms": 7.13791698217392, + "round_gap_p50_ms": 6.308583077043295, + "round_gap_p95_ms": 7.13791698217392, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06429354101419449, + "decode_tps": 170.07173240759667, + "prefill_tps": 824.3440812864679, + "itl_mean_ms": 5.879871897837695, + "itl_p50_ms": 6.303958129137754, + "itl_p95_ms": 7.139707915484905, + "round_gap_p50_ms": 6.330959033221006, + "round_gap_p95_ms": 7.139707915484905, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.920410212403273, + "itl_p50_ms": 6.286084186285734, + "itl_p95_ms": 7.13095860555768, + "round_gap_p50_ms": 6.347040645778179, + "round_gap_p95_ms": 7.132083177566528, + "round_gap_mean_ms": 6.038818416651338, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2494.5150653677015 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 291.1, + "per_stream_tps": 291.1, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 291.141, + "prefill_tps": 2512.884, + "itl_mean_ms": 3.435, + "itl_p50_ms": 3.504, + "itl_p95_ms": 3.961, + "round_gap_p50_ms": 3.524, + "round_gap_p95_ms": 3.961, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2512.884, + "itl_mean_ms": 3.435, + "itl_p50_ms": 3.504, + "itl_p95_ms": 3.961, + "round_gap_p50_ms": 3.524, + "round_gap_p95_ms": 3.961, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 285.2670965447684, + "makespan_s": 0.8974045836366713, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 291.01841572977247, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8977769580669701, + "end_to_end_tps": 285.1487752049252, + "timing_valid": true, + "per_stream_tps": [ + 291.0 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021489250008016825, + "decode_tps": 291.01841572977247, + "prefill_tps": 2512.8843482138577, + "itl_mean_ms": 3.4362086587968994, + "itl_p50_ms": 3.5003330558538437, + "itl_p95_ms": 3.966874908655882, + "round_gap_p50_ms": 3.5255420953035355, + "round_gap_p95_ms": 3.966874908655882, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.4362086587968994, + "itl_p50_ms": 3.5003330558538437, + "itl_p95_ms": 3.966874908655882, + "round_gap_p50_ms": 3.5255420953035355, + "round_gap_p95_ms": 3.966874908655882, + "round_gap_mean_ms": 3.519008867442608, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2512.8843482138577 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 291.1406195137828, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8974045836366713, + "end_to_end_tps": 285.2670965447684, + "timing_valid": true, + "per_stream_tps": [ + 291.1 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021487792022526264, + "decode_tps": 291.1406195137828, + "prefill_tps": 2513.0548519545546, + "itl_mean_ms": 3.434766339612942, + "itl_p50_ms": 3.504292108118534, + "itl_p95_ms": 3.9607081562280655, + "round_gap_p50_ms": 3.51425027474761, + "round_gap_p95_ms": 3.9607081562280655, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.434766339612942, + "itl_p50_ms": 3.504292108118534, + "itl_p95_ms": 3.9607081562280655, + "round_gap_p50_ms": 3.51425027474761, + "round_gap_p95_ms": 3.9607081562280655, + "round_gap_mean_ms": 3.517531793579519, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2513.0548519545546 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 291.3104377660963, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8969214167445898, + "end_to_end_tps": 285.4207684427491, + "timing_valid": true, + "per_stream_tps": [ + 291.3 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02151499968022108, + "decode_tps": 291.3104377660963, + "prefill_tps": 2509.8768674229937, + "itl_mean_ms": 3.432764056339568, + "itl_p50_ms": 3.511791117489338, + "itl_p95_ms": 3.9532920345664024, + "round_gap_p50_ms": 3.524291794747114, + "round_gap_p95_ms": 3.9532920345664024, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.432764056339568, + "itl_p50_ms": 3.511791117489338, + "itl_p95_ms": 3.9532920345664024, + "round_gap_p50_ms": 3.524291794747114, + "round_gap_p95_ms": 3.9532920345664024, + "round_gap_mean_ms": 3.515481262516425, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2509.8768674229937 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 646.9, + "per_stream_tps": 166.7, + "ttft_s_max": 0.09, + "request": { + "ttft_s": 0.054, + "decode_tps": 166.632, + "prefill_tps": 1021.394, + "itl_mean_ms": 6.001, + "itl_p50_ms": 6.594, + "itl_p95_ms": 7.146, + "round_gap_p50_ms": 6.602, + "round_gap_p95_ms": 7.146, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2476.603, + "itl_mean_ms": 6.057, + "itl_p50_ms": 6.589, + "itl_p95_ms": 7.143, + "round_gap_p50_ms": 6.611, + "round_gap_p95_ms": 7.145, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 639.8952170680261, + "makespan_s": 1.6002620002254844, + "steady_tps": 672.8, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 688.6160601062387, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5036039999686182, + "end_to_end_tps": 681.0303776934433, + "timing_valid": true, + "steady_tps": 717.5814153324427, + "per_stream_tps": [ + 177.2, + 179.8, + 174.6, + 172.2 + ], + "ttft_s": [ + 0.06, + 0.08, + 0.04, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.06388187501579523, + "decode_tps": 177.20017466390462, + "prefill_tps": 845.3101914533368, + "itl_mean_ms": 5.643335295220216, + "itl_p50_ms": 5.087333265691996, + "itl_p95_ms": 7.07904202863574, + "round_gap_p50_ms": 5.110208410769701, + "round_gap_p95_ms": 7.07904202863574, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08484762534499168, + "decode_tps": 179.81484372209343, + "prefill_tps": 636.4350184278607, + "itl_mean_ms": 5.561276139947129, + "itl_p50_ms": 5.095541011542082, + "itl_p95_ms": 7.08629097789526, + "round_gap_p50_ms": 5.099959205836058, + "round_gap_p95_ms": 7.08629097789526, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04277699999511242, + "decode_tps": 174.64664383985345, + "prefill_tps": 1262.3606144930661, + "itl_mean_ms": 5.725847219354382, + "itl_p50_ms": 5.106749944388866, + "itl_p95_ms": 7.0852916687726974, + "round_gap_p50_ms": 5.109958816319704, + "round_gap_p95_ms": 7.0852916687726974, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.02185587491840124, + "decode_tps": 172.17165848588834, + "prefill_tps": 2470.731563097274, + "itl_mean_ms": 5.808156863877587, + "itl_p50_ms": 5.100249778479338, + "itl_p95_ms": 7.083083037286997, + "round_gap_p50_ms": 5.106416996568441, + "round_gap_p95_ms": 7.083083037286997, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.6846538795998285, + "itl_p50_ms": 5.095541011542082, + "itl_p95_ms": 7.07904202863574, + "round_gap_p50_ms": 5.108125042170286, + "round_gap_p95_ms": 7.083083037286997, + "round_gap_mean_ms": 5.798346957191825, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2542.305547389008 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 641.0714522632801, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.6150654158554971, + "end_to_end_tps": 634.0300460570442, + "timing_valid": true, + "steady_tps": 666.3116454669571, + "per_stream_tps": [ + 167.0, + 160.3, + 162.5, + 164.7 + ], + "ttft_s": [ + 0.09, + 0.02, + 0.05, + 0.07 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.08722837502136827, + "decode_tps": 166.980496258754, + "prefill_tps": 619.0646104180166, + "itl_mean_ms": 5.98872336832916, + "itl_p50_ms": 6.596666760742664, + "itl_p95_ms": 7.147750351577997, + "round_gap_p50_ms": 6.598332896828651, + "round_gap_p95_ms": 7.147750351577997, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.023362624924629927, + "decode_tps": 160.28518189494858, + "prefill_tps": 2311.3841092004513, + "itl_mean_ms": 6.2388799025439745, + "itl_p50_ms": 6.604541093111038, + "itl_p95_ms": 7.174749858677387, + "round_gap_p50_ms": 6.635540630668402, + "round_gap_p95_ms": 7.174749858677387, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04509479086846113, + "decode_tps": 162.50685837728932, + "prefill_tps": 1197.477556942549, + "itl_mean_ms": 6.153586439277027, + "itl_p50_ms": 6.609250325709581, + "itl_p95_ms": 7.149791345000267, + "round_gap_p50_ms": 6.612500175833702, + "round_gap_p95_ms": 7.149791345000267, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06627399986609817, + "decode_tps": 164.69965404503753, + "prefill_tps": 814.7991687404277, + "itl_mean_ms": 6.071658169522005, + "itl_p50_ms": 6.595958024263382, + "itl_p95_ms": 7.167958188802004, + "round_gap_p50_ms": 6.606541108340025, + "round_gap_p95_ms": 7.167958188802004, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 6.113211969918042, + "itl_p50_ms": 6.598332896828651, + "itl_p95_ms": 7.167958188802004, + "round_gap_p50_ms": 6.611292250454426, + "round_gap_p95_ms": 7.170708384364843, + "round_gap_mean_ms": 6.235476209316403, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2474.3850730359636 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 646.9275540061299, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.6002620002254844, + "end_to_end_tps": 639.8952170680261, + "timing_valid": true, + "steady_tps": 672.7705111496552, + "per_stream_tps": [ + 168.6, + 166.3, + 164.0, + 161.7 + ], + "ttft_s": [ + 0.09, + 0.07, + 0.04, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.08715870790183544, + "decode_tps": 168.59441899996924, + "prefill_tps": 619.559437030879, + "itl_mean_ms": 5.931394443135051, + "itl_p50_ms": 6.575125269591808, + "itl_p95_ms": 7.154458202421665, + "round_gap_p50_ms": 6.579709239304066, + "round_gap_p95_ms": 7.154458202421665, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06604829197749496, + "decode_tps": 166.28443895378908, + "prefill_tps": 817.583595021651, + "itl_mean_ms": 6.013791827375398, + "itl_p50_ms": 6.606040988117456, + "itl_p95_ms": 7.144500035792589, + "round_gap_p50_ms": 6.619249936193228, + "round_gap_p95_ms": 7.144500035792589, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04451725026592612, + "decode_tps": 163.98215995746764, + "prefill_tps": 1213.0129259428238, + "itl_mean_ms": 6.098224345010285, + "itl_p50_ms": 6.594999693334103, + "itl_p95_ms": 7.158916909247637, + "round_gap_p50_ms": 6.612207740545273, + "round_gap_p95_ms": 7.158916909247637, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.023038333281874657, + "decode_tps": 161.74067634741996, + "prefill_tps": 2343.9195596013165, + "itl_mean_ms": 6.182736603944909, + "itl_p50_ms": 6.59362506121397, + "itl_p95_ms": 7.140041328966618, + "round_gap_p50_ms": 6.618209183216095, + "round_gap_p95_ms": 7.140041328966618, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 6.056536804866411, + "itl_p50_ms": 6.5887076780200005, + "itl_p95_ms": 7.142999675124884, + "round_gap_p50_ms": 6.615458056330681, + "round_gap_p95_ms": 7.144500035792589, + "round_gap_mean_ms": 6.177667540963739, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2476.6026935483133 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + } + ] + } + ] + }, + "vl-4-0-old.decode.json": [ + { + "label": "vl-4-0-old", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 293.25580648869806, + "decode_tps_all": [ + 293.32, + 292.64, + 293.26 + ], + "ttft_s_median": 0.02121208282187581, + "prefill_tps_median": 2498.576, + "itl_mean_ms_median": 3.41, + "itl_p50_ms_median": 3.458, + "itl_p95_ms_median": 3.959, + "round_gap_p50_ms_median": 3.476, + "round_gap_p95_ms_median": 3.959, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "sample": "Here's a complete implementation of a red-black tree in Python with detailed explanations and extensive unit tests:\n\n```python\nclass Node:\n def __init__(self" + }, + { + "label": "vl-4-0-old", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 291.50744489745637, + "decode_tps_all": [ + 291.87, + 291.51, + 291.07 + ], + "ttft_s_median": 0.021492457948625088, + "prefill_tps_median": 2512.509, + "itl_mean_ms_median": 3.43, + "itl_p50_ms_median": 3.497, + "itl_p95_ms_median": 3.944, + "round_gap_p50_ms_median": 3.516, + "round_gap_p95_ms_median": 3.944, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "sample": "# Matrix Multiplication on a GPU: A Detailed Tutorial\n\n## Introduction\n\nMatrix multiplication is a fundamental operation in linear algebra, and understanding ho" + } + ], + "vl-4-1-new.concurrent.json": { + "label": "vl-4-1-new", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 301.5, + "per_stream_tps": 301.5, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 301.509, + "prefill_tps": 2516.087, + "itl_mean_ms": 3.317, + "itl_p50_ms": 3.368, + "itl_p95_ms": 3.861, + "round_gap_p50_ms": 3.379, + "round_gap_p95_ms": 3.861, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2516.087, + "itl_mean_ms": 3.317, + "itl_p50_ms": 3.368, + "itl_p95_ms": 3.861, + "round_gap_p50_ms": 3.379, + "round_gap_p95_ms": 3.861, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 295.2957570388391, + "makespan_s": 0.866927457973361, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 301.50927550847507, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.866927457973361, + "end_to_end_tps": 295.2957570388391, + "timing_valid": true, + "per_stream_tps": [ + 301.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02112820791080594, + "decode_tps": 301.50927550847507, + "prefill_tps": 2508.494815260378, + "itl_mean_ms": 3.3166475502737596, + "itl_p50_ms": 3.3676670864224434, + "itl_p95_ms": 3.861249890178442, + "round_gap_p50_ms": 3.380540758371353, + "round_gap_p95_ms": 3.861249890178442, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.3166475502737596, + "itl_p50_ms": 3.3676670864224434, + "itl_p95_ms": 3.861249890178442, + "round_gap_p50_ms": 3.380540758371353, + "round_gap_p95_ms": 3.861249890178442, + "round_gap_mean_ms": 3.3965667683526455, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2508.494815260378 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 301.86655636761316, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8658549156971276, + "end_to_end_tps": 295.6615425505625, + "timing_valid": true, + "per_stream_tps": [ + 301.9 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021059582941234112, + "decode_tps": 301.86655636761316, + "prefill_tps": 2516.669021789002, + "itl_mean_ms": 3.312722058491964, + "itl_p50_ms": 3.363083116710186, + "itl_p95_ms": 3.8611660711467266, + "round_gap_p50_ms": 3.373874817043543, + "round_gap_p95_ms": 3.8611660711467266, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.312722058491964, + "itl_p50_ms": 3.363083116710186, + "itl_p95_ms": 3.8611660711467266, + "round_gap_p50_ms": 3.373874817043543, + "round_gap_p95_ms": 3.8611660711467266, + "round_gap_mean_ms": 3.392546686407433, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2516.669021789002 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 301.2526796646745, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8675809581764042, + "end_to_end_tps": 295.07332726399903, + "timing_valid": true, + "per_stream_tps": [ + 301.3 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021064458414912224, + "decode_tps": 301.2526796646745, + "prefill_tps": 2516.086526225595, + "itl_mean_ms": 3.319472547474445, + "itl_p50_ms": 3.373540937900543, + "itl_p95_ms": 3.867959138005972, + "round_gap_p50_ms": 3.379375208169222, + "round_gap_p95_ms": 3.867959138005972, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.319472547474445, + "itl_p50_ms": 3.373540937900543, + "itl_p95_ms": 3.867959138005972, + "round_gap_p50_ms": 3.379375208169222, + "round_gap_p95_ms": 3.867959138005972, + "round_gap_mean_ms": 3.399459837775034, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2516.086526225595 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 813.4, + "per_stream_tps": 208.5, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.052, + "decode_tps": 208.474, + "prefill_tps": 1055.39, + "itl_mean_ms": 4.797, + "itl_p50_ms": 4.73, + "itl_p95_ms": 5.168, + "round_gap_p50_ms": 4.74, + "round_gap_p95_ms": 5.168, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2548.794, + "itl_mean_ms": 4.796, + "itl_p50_ms": 4.73, + "itl_p95_ms": 5.17, + "round_gap_p50_ms": 4.745, + "round_gap_p95_ms": 5.171, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 802.6079530340896, + "makespan_s": 1.2758408337831497, + "steady_tps": 852.9, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 812.5775590578769, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2771217497065663, + "end_to_end_tps": 801.8029606302422, + "timing_valid": true, + "steady_tps": 852.0472488867083, + "per_stream_tps": [ + 213.7, + 206.5, + 203.2, + 210.1 + ], + "ttft_s": [ + 0.08, + 0.04, + 0.02, + 0.06 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.08314141677692533, + "decode_tps": 213.67115541197853, + "prefill_tps": 637.4680881636043, + "itl_mean_ms": 4.68008888739289, + "itl_p50_ms": 4.726666957139969, + "itl_p95_ms": 5.165582988411188, + "round_gap_p50_ms": 4.729457665234804, + "round_gap_p95_ms": 5.165582988411188, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04190258402377367, + "decode_tps": 206.5418208142914, + "prefill_tps": 1264.838463659667, + "itl_mean_ms": 4.84163447411037, + "itl_p50_ms": 4.737833980470896, + "itl_p95_ms": 5.1817079074680805, + "round_gap_p50_ms": 4.740958102047443, + "round_gap_p95_ms": 5.1817079074680805, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021204666700214148, + "decode_tps": 203.15010121666265, + "prefill_tps": 2499.449802692006, + "itl_mean_ms": 4.922468627930856, + "itl_p50_ms": 4.732916131615639, + "itl_p95_ms": 5.165374837815762, + "round_gap_p50_ms": 4.738542251288891, + "round_gap_p95_ms": 5.165374837815762, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06259104190394282, + "decode_tps": 210.05186732318134, + "prefill_tps": 846.7665401917739, + "itl_mean_ms": 4.760728922544741, + "itl_p50_ms": 4.734250251203775, + "itl_p95_ms": 5.1782079972326756, + "round_gap_p50_ms": 4.755458328872919, + "round_gap_p95_ms": 5.1782079972326756, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.801230227994714, + "itl_p50_ms": 4.732916131615639, + "itl_p95_ms": 5.173375364392996, + "round_gap_p50_ms": 4.7455839812755585, + "round_gap_p95_ms": 5.175500176846981, + "round_gap_mean_ms": 4.897254832554609, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2548.753423944623 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 813.6630243922096, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2749731251969934, + "end_to_end_tps": 803.154184008219, + "timing_valid": true, + "steady_tps": 853.5567008951057, + "per_stream_tps": [ + 206.9, + 203.5, + 214.0, + 210.4 + ], + "ttft_s": [ + 0.04, + 0.02, + 0.08, + 0.06 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.041903417091816664, + "decode_tps": 206.89588737753647, + "prefill_tps": 1264.8133178224837, + "itl_mean_ms": 4.833348853257942, + "itl_p50_ms": 4.725834354758263, + "itl_p95_ms": 5.15670794993639, + "round_gap_p50_ms": 4.734166897833347, + "round_gap_p95_ms": 5.15670794993639, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.02127866679802537, + "decode_tps": 203.4932204531743, + "prefill_tps": 2490.757550887461, + "itl_mean_ms": 4.914168628188325, + "itl_p50_ms": 4.721708130091429, + "itl_p95_ms": 5.165459122508764, + "round_gap_p50_ms": 4.726333077996969, + "round_gap_p95_ms": 5.165459122508764, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08305616723373532, + "decode_tps": 213.9675050742759, + "prefill_tps": 638.122390729255, + "itl_mean_ms": 4.673606862186217, + "itl_p50_ms": 4.714457783848047, + "itl_p95_ms": 5.165750160813332, + "round_gap_p50_ms": 4.721791949123144, + "round_gap_p95_ms": 5.165750160813332, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06254066620022058, + "decode_tps": 210.4023687214775, + "prefill_tps": 847.4485997690422, + "itl_mean_ms": 4.752798203159781, + "itl_p50_ms": 4.731959197670221, + "itl_p95_ms": 5.168459378182888, + "round_gap_p50_ms": 4.74158301949501, + "round_gap_p95_ms": 5.168459378182888, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.7934806366980665, + "itl_p50_ms": 4.721791949123144, + "itl_p95_ms": 5.165040958672762, + "round_gap_p50_ms": 4.734875168651342, + "round_gap_p95_ms": 5.165459122508764, + "round_gap_mean_ms": 4.889350249432027, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2548.7942906752305 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 813.406534628959, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2758408337831497, + "end_to_end_tps": 802.6079530340896, + "timing_valid": true, + "steady_tps": 852.9414869130479, + "per_stream_tps": [ + 203.4, + 210.3, + 213.9, + 206.8 + ], + "ttft_s": [ + 0.02, + 0.06, + 0.08, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021292333956807852, + "decode_tps": 203.3584381150898, + "prefill_tps": 2489.1587792823516, + "itl_mean_ms": 4.917425651322393, + "itl_p50_ms": 4.729124717414379, + "itl_p95_ms": 5.170916672796011, + "round_gap_p50_ms": 4.738125018775463, + "round_gap_p95_ms": 5.170916672796011, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.062491791788488626, + "decode_tps": 210.26834457229208, + "prefill_tps": 848.1113836419541, + "itl_mean_ms": 4.755827616534981, + "itl_p50_ms": 4.730167333036661, + "itl_p95_ms": 5.171374883502722, + "round_gap_p50_ms": 4.748541861772537, + "round_gap_p95_ms": 5.171374883502722, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08299479214474559, + "decode_tps": 213.89420396377514, + "prefill_tps": 638.5942856217567, + "itl_mean_ms": 4.675208497792482, + "itl_p50_ms": 4.724125377833843, + "itl_p95_ms": 5.173708777874708, + "round_gap_p50_ms": 4.7478750348091125, + "round_gap_p95_ms": 5.173708777874708, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04197462508454919, + "decode_tps": 206.75934826093712, + "prefill_tps": 1262.6676210506341, + "itl_mean_ms": 4.8365406856379085, + "itl_p50_ms": 4.737375304102898, + "itl_p95_ms": 5.168458912521601, + "round_gap_p50_ms": 4.74433321505785, + "round_gap_p95_ms": 5.168458912521601, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.796250612821941, + "itl_p50_ms": 4.730167333036661, + "itl_p95_ms": 5.170249845832586, + "round_gap_p50_ms": 4.744708072394133, + "round_gap_p95_ms": 5.170916672796011, + "round_gap_mean_ms": 4.89217562507838, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2549.711339899949 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 299.8, + "per_stream_tps": 299.8, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 299.83, + "prefill_tps": 2530.706, + "itl_mean_ms": 3.335, + "itl_p50_ms": 3.403, + "itl_p95_ms": 3.864, + "round_gap_p50_ms": 3.417, + "round_gap_p95_ms": 3.864, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2530.706, + "itl_mean_ms": 3.335, + "itl_p50_ms": 3.403, + "itl_p95_ms": 3.864, + "round_gap_p50_ms": 3.417, + "round_gap_p95_ms": 3.864, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 293.61837358393734, + "makespan_s": 0.8718800423666835, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 299.6057541180388, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8725046250037849, + "end_to_end_tps": 293.40818680346763, + "timing_valid": true, + "per_stream_tps": [ + 299.6 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021337249781936407, + "decode_tps": 299.6057541180388, + "prefill_tps": 2530.785389488906, + "itl_mean_ms": 3.3377196073678195, + "itl_p50_ms": 3.4032082185149193, + "itl_p95_ms": 3.864208236336708, + "round_gap_p50_ms": 3.422374837100506, + "round_gap_p95_ms": 3.864208236336708, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.3377196073678195, + "itl_p50_ms": 3.4032082185149193, + "itl_p95_ms": 3.864208236336708, + "round_gap_p50_ms": 3.422374837100506, + "round_gap_p95_ms": 3.864208236336708, + "round_gap_mean_ms": 3.41814658585861, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2530.785389488906 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 300.5036676219269, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8699639160186052, + "end_to_end_tps": 294.2650784547311, + "timing_valid": true, + "per_stream_tps": [ + 300.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021337916143238544, + "decode_tps": 300.5036676219269, + "prefill_tps": 2530.7063556490384, + "itl_mean_ms": 3.327746406270593, + "itl_p50_ms": 3.3883750438690186, + "itl_p95_ms": 3.846375271677971, + "round_gap_p50_ms": 3.4139580093324184, + "round_gap_p95_ms": 3.846375271677971, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.327746406270593, + "itl_p50_ms": 3.3883750438690186, + "itl_p95_ms": 3.846375271677971, + "round_gap_p50_ms": 3.4139580093324184, + "round_gap_p95_ms": 3.846375271677971, + "round_gap_mean_ms": 3.4079330666626553, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2530.7063556490384 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 299.8304487561572, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8718800423666835, + "end_to_end_tps": 293.61837358393734, + "timing_valid": true, + "per_stream_tps": [ + 299.8 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021349542308598757, + "decode_tps": 299.8304487561572, + "prefill_tps": 2529.3282272496735, + "itl_mean_ms": 3.3352183013716163, + "itl_p50_ms": 3.4054582938551903, + "itl_p95_ms": 3.8755410350859165, + "round_gap_p50_ms": 3.416541963815689, + "round_gap_p95_ms": 3.8755410350859165, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.3352183013716163, + "itl_p50_ms": 3.4054582938551903, + "itl_p95_ms": 3.8755410350859165, + "round_gap_p50_ms": 3.416541963815689, + "round_gap_p95_ms": 3.8755410350859165, + "round_gap_mean_ms": 3.415585007428764, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2529.3282272496735 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 810.7, + "per_stream_tps": 207.9, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.053, + "decode_tps": 207.967, + "prefill_tps": 1060.764, + "itl_mean_ms": 4.809, + "itl_p50_ms": 4.757, + "itl_p95_ms": 5.169, + "round_gap_p50_ms": 4.765, + "round_gap_p95_ms": 5.169, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2561.345, + "itl_mean_ms": 4.811, + "itl_p50_ms": 4.756, + "itl_p95_ms": 5.171, + "round_gap_p50_ms": 4.767, + "round_gap_p95_ms": 5.172, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 799.885771391415, + "makespan_s": 1.2801827918738127, + "steady_tps": 851.2, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 811.3790068018363, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2791504156775773, + "end_to_end_tps": 800.5313428738388, + "timing_valid": true, + "steady_tps": 852.0150809654127, + "per_stream_tps": [ + 209.8, + 202.9, + 213.5, + 206.3 + ], + "ttft_s": [ + 0.06, + 0.02, + 0.08, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.06341308308765292, + "decode_tps": 209.84886557837106, + "prefill_tps": 851.559289829172, + "itl_mean_ms": 4.765334314502337, + "itl_p50_ms": 4.760041832923889, + "itl_p95_ms": 5.164541769772768, + "round_gap_p50_ms": 4.772000014781952, + "round_gap_p95_ms": 5.164541769772768, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021532165817916393, + "decode_tps": 202.8702828489376, + "prefill_tps": 2507.8759125599854, + "itl_mean_ms": 4.929258174025545, + "itl_p50_ms": 4.762166645377874, + "itl_p95_ms": 5.16725005581975, + "round_gap_p50_ms": 4.763749893754721, + "round_gap_p95_ms": 5.16725005581975, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08421695791184902, + "decode_tps": 213.4964202844186, + "prefill_tps": 641.2010281411791, + "itl_mean_ms": 4.683919283835327, + "itl_p50_ms": 4.744249861687422, + "itl_p95_ms": 5.160292144864798, + "round_gap_p50_ms": 4.748999606817961, + "round_gap_p95_ms": 5.160292144864798, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04255383275449276, + "decode_tps": 206.30460811961356, + "prefill_tps": 1268.9808768000757, + "itl_mean_ms": 4.847201471235238, + "itl_p50_ms": 4.756374750286341, + "itl_p95_ms": 5.165833048522472, + "round_gap_p50_ms": 4.765375051647425, + "round_gap_p95_ms": 5.165833048522472, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.806428310899612, + "itl_p50_ms": 4.752874840050936, + "itl_p95_ms": 5.164582747966051, + "round_gap_p50_ms": 4.765375051647425, + "round_gap_p95_ms": 5.164625123143196, + "round_gap_mean_ms": 4.902556877117604, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2561.1066810758034 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 810.6569065454623, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2803308754228055, + "end_to_end_tps": 799.7932563032529, + "timing_valid": true, + "steady_tps": 851.1152607913817, + "per_stream_tps": [ + 206.1, + 209.6, + 213.3, + 202.7 + ], + "ttft_s": [ + 0.04, + 0.06, + 0.08, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.04245595773681998, + "decode_tps": 206.091311514682, + "prefill_tps": 1271.906297220765, + "itl_mean_ms": 4.852218138894029, + "itl_p50_ms": 4.772000014781952, + "itl_p95_ms": 5.166542250663042, + "round_gap_p50_ms": 4.775292240083218, + "round_gap_p95_ms": 5.166542250663042, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06333966599777341, + "decode_tps": 209.6288560642435, + "prefill_tps": 852.5463333181813, + "itl_mean_ms": 4.770335624469261, + "itl_p50_ms": 4.756666719913483, + "itl_p95_ms": 5.171999800950289, + "round_gap_p50_ms": 4.767791833728552, + "round_gap_p95_ms": 5.171999800950289, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08415970904752612, + "decode_tps": 213.28922950999976, + "prefill_tps": 641.637199214953, + "itl_mean_ms": 4.688469278534837, + "itl_p50_ms": 4.753291606903076, + "itl_p95_ms": 5.179791711270809, + "round_gap_p50_ms": 4.7642081044614315, + "round_gap_p95_ms": 5.179791711270809, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021617917343974113, + "decode_tps": 202.67322683521115, + "prefill_tps": 2497.9279521138624, + "itl_mean_ms": 4.934050814778197, + "itl_p50_ms": 4.7561246901750565, + "itl_p95_ms": 5.174875259399414, + "round_gap_p50_ms": 4.767291713505983, + "round_gap_p95_ms": 5.174875259399414, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.811268464169082, + "itl_p50_ms": 4.7561246901750565, + "itl_p95_ms": 5.174875259399414, + "round_gap_p50_ms": 4.770874977111816, + "round_gap_p95_ms": 5.1749167032539845, + "round_gap_mean_ms": 4.907493833452463, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2562.366206647741 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 810.6135811783071, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2801827918738127, + "end_to_end_tps": 799.885771391415, + "timing_valid": true, + "steady_tps": 851.2348217704872, + "per_stream_tps": [ + 202.7, + 213.3, + 206.1, + 209.7 + ], + "ttft_s": [ + 0.02, + 0.08, + 0.04, + 0.06 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02154254214838147, + "decode_tps": 202.68934309683755, + "prefill_tps": 2506.667951630635, + "itl_mean_ms": 4.933658497882824, + "itl_p50_ms": 4.7565000131726265, + "itl_p95_ms": 5.171624943614006, + "round_gap_p50_ms": 4.762125201523304, + "round_gap_p95_ms": 5.171624943614006, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.0842609996907413, + "decode_tps": 213.28934099605468, + "prefill_tps": 640.8658833647044, + "itl_mean_ms": 4.68846682787818, + "itl_p50_ms": 4.756166134029627, + "itl_p95_ms": 5.1665413193404675, + "round_gap_p50_ms": 4.761125426739454, + "round_gap_p95_ms": 5.1665413193404675, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04237762512639165, + "decode_tps": 206.11665314619452, + "prefill_tps": 1274.2573430895316, + "itl_mean_ms": 4.851621568349063, + "itl_p50_ms": 4.766374826431274, + "itl_p95_ms": 5.1788748241961, + "round_gap_p50_ms": 4.77454112842679, + "round_gap_p95_ms": 5.1788748241961, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06342833396047354, + "decode_tps": 209.65899690411652, + "prefill_tps": 851.3545387090103, + "itl_mean_ms": 4.769649835047769, + "itl_p50_ms": 4.7600422985851765, + "itl_p95_ms": 5.171750206500292, + "round_gap_p50_ms": 4.764291923493147, + "round_gap_p95_ms": 5.171750206500292, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.810849182289459, + "itl_p50_ms": 4.7593750059604645, + "itl_p95_ms": 5.170708056539297, + "round_gap_p50_ms": 4.767249803990126, + "round_gap_p95_ms": 5.171624943614006, + "round_gap_mean_ms": 4.907066165935248, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2561.3445360421956 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + } + ] + } + ] + }, + "vl-4-1-new.decode.json": [ + { + "label": "vl-4-1-new", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 301.64586239108235, + "decode_tps_all": [ + 302.49, + 301.65, + 301.54 + ], + "ttft_s_median": 0.021058999933302402, + "prefill_tps_median": 2516.739, + "itl_mean_ms_median": 3.315, + "itl_p50_ms_median": 3.368, + "itl_p95_ms_median": 3.865, + "round_gap_p50_ms_median": 3.38, + "round_gap_p95_ms_median": 3.865, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "sample": "Here's a complete implementation of a red-black tree in Python with detailed explanations and extensive unit tests:\n\n```python\nclass Node:\n def __init__(self" + }, + { + "label": "vl-4-1-new", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 299.8096793417327, + "decode_tps_all": [ + 301.25, + 299.81, + 299.49 + ], + "ttft_s_median": 0.021359582897275686, + "prefill_tps_median": 2528.139, + "itl_mean_ms_median": 3.335, + "itl_p50_ms_median": 3.4, + "itl_p95_ms_median": 3.868, + "round_gap_p50_ms_median": 3.414, + "round_gap_p95_ms_median": 3.868, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "sample": "# Matrix Multiplication on a GPU: A Detailed Tutorial\n\n## Introduction\n\nMatrix multiplication is a fundamental operation in linear algebra, and understanding ho" + } + ], + "vl-4-2-new.concurrent.json": { + "label": "vl-4-2-new", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 301.8, + "per_stream_tps": 301.8, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 301.759, + "prefill_tps": 2516.33, + "itl_mean_ms": 3.314, + "itl_p50_ms": 3.366, + "itl_p95_ms": 3.867, + "round_gap_p50_ms": 3.381, + "round_gap_p95_ms": 3.867, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2516.33, + "itl_mean_ms": 3.314, + "itl_p50_ms": 3.366, + "itl_p95_ms": 3.867, + "round_gap_p50_ms": 3.381, + "round_gap_p95_ms": 3.867, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 295.5431429376609, + "makespan_s": 0.8662017919123173, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 301.8784982191917, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8658280419185758, + "end_to_end_tps": 295.67071936447485, + "timing_valid": true, + "per_stream_tps": [ + 301.9 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021062416955828667, + "decode_tps": 301.8784982191917, + "prefill_tps": 2516.330396039052, + "itl_mean_ms": 3.312591012275102, + "itl_p50_ms": 3.365791868418455, + "itl_p95_ms": 3.857792355120182, + "round_gap_p50_ms": 3.3817081712186337, + "round_gap_p95_ms": 3.857792355120182, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.312591012275102, + "itl_p50_ms": 3.365791868418455, + "itl_p95_ms": 3.857792355120182, + "round_gap_p50_ms": 3.3817081712186337, + "round_gap_p95_ms": 3.857792355120182, + "round_gap_mean_ms": 3.3924124824504056, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2516.330396039052 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 301.75851227548924, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8662017919123173, + "end_to_end_tps": 295.5431429376609, + "timing_valid": true, + "per_stream_tps": [ + 301.8 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021102124825119972, + "decode_tps": 301.75851227548924, + "prefill_tps": 2511.595417012641, + "itl_mean_ms": 3.313908172661767, + "itl_p50_ms": 3.3647906966507435, + "itl_p95_ms": 3.866916988044977, + "round_gap_p50_ms": 3.3749998547136784, + "round_gap_p95_ms": 3.866916988044977, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.313908172661767, + "itl_p50_ms": 3.3647906966507435, + "itl_p95_ms": 3.866916988044977, + "round_gap_p50_ms": 3.3749998547136784, + "round_gap_p95_ms": 3.866916988044977, + "round_gap_mean_ms": 3.393761381641569, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2511.595417012641 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 300.86378868612405, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8687031250447035, + "end_to_end_tps": 294.69215963373705, + "timing_valid": true, + "per_stream_tps": [ + 300.9 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021056165918707848, + "decode_tps": 300.86378868612405, + "prefill_tps": 2517.0774301750207, + "itl_mean_ms": 3.3237632364034653, + "itl_p50_ms": 3.374749794602394, + "itl_p95_ms": 3.8761249743402004, + "round_gap_p50_ms": 3.380874637514353, + "round_gap_p95_ms": 3.8761249743402004, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.3237632364034653, + "itl_p50_ms": 3.374749794602394, + "itl_p95_ms": 3.8761249743402004, + "round_gap_p50_ms": 3.380874637514353, + "round_gap_p95_ms": 3.8761249743402004, + "round_gap_mean_ms": 3.4038539167987296, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2517.0774301750207 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 813.4, + "per_stream_tps": 208.4, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.052, + "decode_tps": 208.45, + "prefill_tps": 1054.336, + "itl_mean_ms": 4.798, + "itl_p50_ms": 4.73, + "itl_p95_ms": 5.174, + "round_gap_p50_ms": 4.74, + "round_gap_p95_ms": 5.174, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2545.121, + "itl_mean_ms": 4.796, + "itl_p50_ms": 4.728, + "itl_p95_ms": 5.172, + "round_gap_p50_ms": 4.741, + "round_gap_p95_ms": 5.174, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 802.4412023181818, + "makespan_s": 1.276105958968401, + "steady_tps": 853.0, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 813.3778863749797, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.276105958968401, + "end_to_end_tps": 802.4412023181818, + "timing_valid": true, + "steady_tps": 852.9681092128834, + "per_stream_tps": [ + 203.4, + 210.3, + 206.8, + 213.9 + ], + "ttft_s": [ + 0.02, + 0.06, + 0.04, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02142829168587923, + "decode_tps": 203.35261345116947, + "prefill_tps": 2473.3656222780382, + "itl_mean_ms": 4.917566501992006, + "itl_p50_ms": 4.722374957054853, + "itl_p95_ms": 5.175833124667406, + "round_gap_p50_ms": 4.74195834249258, + "round_gap_p95_ms": 5.175833124667406, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06262116599828005, + "decode_tps": 210.26679861329603, + "prefill_tps": 846.3592006807362, + "itl_mean_ms": 4.755862583132351, + "itl_p50_ms": 4.731208086013794, + "itl_p95_ms": 5.178582854568958, + "round_gap_p50_ms": 4.739750176668167, + "round_gap_p95_ms": 5.178582854568958, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04203554103150964, + "decode_tps": 206.75385804598523, + "prefill_tps": 1260.8378219819142, + "itl_mean_ms": 4.836669116847071, + "itl_p50_ms": 4.733665846288204, + "itl_p95_ms": 5.172333214432001, + "round_gap_p50_ms": 4.740207921713591, + "round_gap_p95_ms": 5.172333214432001, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08336825016885996, + "decode_tps": 213.89939959289393, + "prefill_tps": 635.7336263223715, + "itl_mean_ms": 4.675094936700428, + "itl_p50_ms": 4.72270930185914, + "itl_p95_ms": 5.170291755348444, + "round_gap_p50_ms": 4.734625108540058, + "round_gap_p95_ms": 5.170291755348444, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.796298284667964, + "itl_p50_ms": 4.7278329730033875, + "itl_p95_ms": 5.17187500372529, + "round_gap_p50_ms": 4.741000011563301, + "round_gap_p95_ms": 5.172333214432001, + "round_gap_mean_ms": 4.892224250361323, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2542.934505289486 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 813.5297981771797, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2757671251893044, + "end_to_end_tps": 802.6543244309215, + "timing_valid": true, + "steady_tps": 853.106613361527, + "per_stream_tps": [ + 206.8, + 210.3, + 213.9, + 203.4 + ], + "ttft_s": [ + 0.04, + 0.06, + 0.08, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.04204925009980798, + "decode_tps": 206.79394472674988, + "prefill_tps": 1260.426758484381, + "itl_mean_ms": 4.835731536149979, + "itl_p50_ms": 4.727916792035103, + "itl_p95_ms": 5.172292236238718, + "round_gap_p50_ms": 4.731667228043079, + "round_gap_p95_ms": 5.172292236238718, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06260195793583989, + "decode_tps": 210.3027091125054, + "prefill_tps": 846.618887772155, + "itl_mean_ms": 4.755050489934636, + "itl_p50_ms": 4.723458085209131, + "itl_p95_ms": 5.168790929019451, + "round_gap_p50_ms": 4.725208040326834, + "round_gap_p95_ms": 5.168790929019451, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08317016577348113, + "decode_tps": 213.93347298931886, + "prefill_tps": 637.2477379009756, + "itl_mean_ms": 4.674350329693041, + "itl_p50_ms": 4.728792235255241, + "itl_p95_ms": 5.175832659006119, + "round_gap_p50_ms": 4.7337086871266365, + "round_gap_p95_ms": 5.175832659006119, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021460459101945162, + "decode_tps": 203.39118911607133, + "prefill_tps": 2469.658256062011, + "itl_mean_ms": 4.91663382443435, + "itl_p50_ms": 4.720249678939581, + "itl_p95_ms": 5.17925014719367, + "round_gap_p50_ms": 4.724292084574699, + "round_gap_p95_ms": 5.17925014719367, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.795441545053002, + "itl_p50_ms": 4.724083002656698, + "itl_p95_ms": 5.172292236238718, + "round_gap_p50_ms": 4.728792235255241, + "round_gap_p95_ms": 5.173957906663418, + "round_gap_mean_ms": 4.891350375954062, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2545.1211275788087 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 812.8035686081696, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2767799999564886, + "end_to_end_tps": 802.0175754906068, + "timing_valid": true, + "steady_tps": 852.2787287825442, + "per_stream_tps": [ + 203.2, + 213.7, + 210.1, + 206.6 + ], + "ttft_s": [ + 0.02, + 0.08, + 0.06, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021320332773029804, + "decode_tps": 203.20548007245677, + "prefill_tps": 2485.8899044505038, + "itl_mean_ms": 4.921127125328662, + "itl_p50_ms": 4.730792250484228, + "itl_p95_ms": 5.173500161617994, + "round_gap_p50_ms": 4.745499696582556, + "round_gap_p95_ms": 5.173500161617994, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08311591716483235, + "decode_tps": 213.73044922894704, + "prefill_tps": 637.663660678765, + "itl_mean_ms": 4.678790521461005, + "itl_p50_ms": 4.732457920908928, + "itl_p95_ms": 5.168125033378601, + "round_gap_p50_ms": 4.74216602742672, + "round_gap_p95_ms": 5.168125033378601, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06248200032860041, + "decode_tps": 210.10626946688328, + "prefill_tps": 848.2442898957553, + "itl_mean_ms": 4.7594962422462075, + "itl_p50_ms": 4.746375139802694, + "itl_p95_ms": 5.1753344014286995, + "round_gap_p50_ms": 4.755374975502491, + "round_gap_p95_ms": 5.1753344014286995, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.041976958978921175, + "decode_tps": 206.5975998885926, + "prefill_tps": 1262.5974174692853, + "itl_mean_ms": 4.840327286179743, + "itl_p50_ms": 4.734625108540058, + "itl_p95_ms": 5.1791248843073845, + "round_gap_p50_ms": 4.741041921079159, + "round_gap_p95_ms": 5.1791248843073845, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.799935293803904, + "itl_p50_ms": 4.733417183160782, + "itl_p95_ms": 5.173875018954277, + "round_gap_p50_ms": 4.750874824821949, + "round_gap_p95_ms": 5.174207966774702, + "round_gap_mean_ms": 4.895933999679983, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2548.4163947395677 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 299.0, + "per_stream_tps": 299.0, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 299.034, + "prefill_tps": 2520.739, + "itl_mean_ms": 3.344, + "itl_p50_ms": 3.411, + "itl_p95_ms": 3.876, + "round_gap_p50_ms": 3.433, + "round_gap_p95_ms": 3.876, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2520.739, + "itl_mean_ms": 3.344, + "itl_p50_ms": 3.411, + "itl_p95_ms": 3.876, + "round_gap_p50_ms": 3.433, + "round_gap_p95_ms": 3.876, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 292.83041617239707, + "makespan_s": 0.8742261249572039, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 299.22390997501043, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.873691916000098, + "end_to_end_tps": 293.0094639904752, + "timing_valid": true, + "per_stream_tps": [ + 299.2 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021422290708869696, + "decode_tps": 299.22390997501043, + "prefill_tps": 2520.7388291879456, + "itl_mean_ms": 3.341978921682811, + "itl_p50_ms": 3.4044161438941956, + "itl_p95_ms": 3.876333124935627, + "round_gap_p50_ms": 3.415624611079693, + "round_gap_p95_ms": 3.876333124935627, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.341978921682811, + "itl_p50_ms": 3.4044161438941956, + "itl_p95_ms": 3.876333124935627, + "round_gap_p50_ms": 3.415624611079693, + "round_gap_p95_ms": 3.876333124935627, + "round_gap_mean_ms": 3.4225085342534816, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2520.7388291879456 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 298.86612546702355, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8746760413050652, + "end_to_end_tps": 292.6797899003085, + "timing_valid": true, + "per_stream_tps": [ + 298.9 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02139870822429657, + "decode_tps": 298.86612546702355, + "prefill_tps": 2523.5168139115613, + "itl_mean_ms": 3.345979737373544, + "itl_p50_ms": 3.410500008612871, + "itl_p95_ms": 3.8668341003358364, + "round_gap_p50_ms": 3.433250356465578, + "round_gap_p95_ms": 3.8668341003358364, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.345979737373544, + "itl_p50_ms": 3.410500008612871, + "itl_p95_ms": 3.8668341003358364, + "round_gap_p50_ms": 3.433250356465578, + "round_gap_p95_ms": 3.8668341003358364, + "round_gap_mean_ms": 3.4266057551415807, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2523.5168139115613 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 299.03356461835216, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8742261249572039, + "end_to_end_tps": 292.83041617239707, + "timing_valid": true, + "per_stream_tps": [ + 299.0 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021428042091429234, + "decode_tps": 299.03356461835216, + "prefill_tps": 2520.0622515856858, + "itl_mean_ms": 3.3441062085330486, + "itl_p50_ms": 3.4120408818125725, + "itl_p95_ms": 3.8805417716503143, + "round_gap_p50_ms": 3.4352093935012817, + "round_gap_p95_ms": 3.8805417716503143, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.3441062085330486, + "itl_p50_ms": 3.4120408818125725, + "itl_p95_ms": 3.8805417716503143, + "round_gap_p50_ms": 3.4352093935012817, + "round_gap_p95_ms": 3.8805417716503143, + "round_gap_mean_ms": 3.424687081027821, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2520.0622515856858 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 809.5, + "per_stream_tps": 207.5, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.053, + "decode_tps": 207.495, + "prefill_tps": 1060.502, + "itl_mean_ms": 4.82, + "itl_p50_ms": 4.775, + "itl_p95_ms": 5.18, + "round_gap_p50_ms": 4.782, + "round_gap_p95_ms": 5.18, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2557.777, + "itl_mean_ms": 4.818, + "itl_p50_ms": 4.776, + "itl_p95_ms": 5.178, + "round_gap_p50_ms": 4.782, + "round_gap_p95_ms": 5.178, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 798.643345839968, + "makespan_s": 1.2821743339300156, + "steady_tps": 849.9, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 809.5129143606528, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2821514159440994, + "end_to_end_tps": 798.657621296614, + "timing_valid": true, + "steady_tps": 849.9299773213114, + "per_stream_tps": [ + 213.0, + 205.8, + 209.3, + 202.4 + ], + "ttft_s": [ + 0.08, + 0.04, + 0.06, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.08442999981343746, + "decode_tps": 212.99308643593386, + "prefill_tps": 639.5830879938676, + "itl_mean_ms": 4.6949880708958585, + "itl_p50_ms": 4.76366700604558, + "itl_p95_ms": 5.173750221729279, + "round_gap_p50_ms": 4.776417277753353, + "round_gap_p95_ms": 5.173750221729279, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04254170786589384, + "decode_tps": 205.80466668768076, + "prefill_tps": 1269.3425513199106, + "itl_mean_ms": 4.85897631037469, + "itl_p50_ms": 4.778583999723196, + "itl_p95_ms": 5.167415831238031, + "round_gap_p50_ms": 4.781041759997606, + "round_gap_p95_ms": 5.167415831238031, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06341579230502248, + "decode_tps": 209.3396116105729, + "prefill_tps": 851.5229099443617, + "itl_mean_ms": 4.776926795203312, + "itl_p50_ms": 4.777417052537203, + "itl_p95_ms": 5.174791906028986, + "round_gap_p50_ms": 4.7784168273210526, + "round_gap_p95_ms": 5.174791906028986, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.02167112473398447, + "decode_tps": 202.38641360432723, + "prefill_tps": 2491.794988163105, + "itl_mean_ms": 4.94104313718922, + "itl_p50_ms": 4.778292030096054, + "itl_p95_ms": 5.178667139261961, + "round_gap_p50_ms": 4.782082978636026, + "round_gap_p95_ms": 5.178667139261961, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.81798357841577, + "itl_p50_ms": 4.775958135724068, + "itl_p95_ms": 5.174166988581419, + "round_gap_p50_ms": 4.782082978636026, + "round_gap_p95_ms": 5.1743341609835625, + "round_gap_mean_ms": 4.914343249984086, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2557.23818625081 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 809.533151898022, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2821743339300156, + "end_to_end_tps": 798.643345839968, + "timing_valid": true, + "steady_tps": 849.9078217111008, + "per_stream_tps": [ + 209.3, + 205.8, + 213.0, + 202.4 + ], + "ttft_s": [ + 0.06, + 0.04, + 0.08, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.06348287500441074, + "decode_tps": 209.33647526649298, + "prefill_tps": 850.6231010528134, + "itl_mean_ms": 4.776998364603987, + "itl_p50_ms": 4.768334329128265, + "itl_p95_ms": 5.177665967494249, + "round_gap_p50_ms": 4.772916901856661, + "round_gap_p95_ms": 5.177665967494249, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.0425501661375165, + "decode_tps": 205.8033241167443, + "prefill_tps": 1269.0902269448056, + "itl_mean_ms": 4.859008008212435, + "itl_p50_ms": 4.7622923739254475, + "itl_p95_ms": 5.174583289772272, + "round_gap_p50_ms": 4.766499623656273, + "round_gap_p95_ms": 5.174583289772272, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.0843195840716362, + "decode_tps": 212.99158904147922, + "prefill_tps": 640.4206163318204, + "itl_mean_ms": 4.695021078063577, + "itl_p50_ms": 4.7661662101745605, + "itl_p95_ms": 5.181000102311373, + "round_gap_p50_ms": 4.769457969814539, + "round_gap_p95_ms": 5.181000102311373, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021693249698728323, + "decode_tps": 202.38781234521758, + "prefill_tps": 2489.2536042290394, + "itl_mean_ms": 4.941008988694818, + "itl_p50_ms": 4.773292224854231, + "itl_p95_ms": 5.18366601318121, + "round_gap_p50_ms": 4.784625023603439, + "round_gap_p95_ms": 5.18366601318121, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.818009109893704, + "itl_p50_ms": 4.766250029206276, + "itl_p95_ms": 5.177665967494249, + "round_gap_p50_ms": 4.773708991706371, + "round_gap_p95_ms": 5.177707877010107, + "round_gap_mean_ms": 4.914369292091578, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2557.776929450609 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 808.8545518253666, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2829905417747796, + "end_to_end_tps": 798.1352680772578, + "timing_valid": true, + "steady_tps": 849.2940440354105, + "per_stream_tps": [ + 202.2, + 205.7, + 209.2, + 212.8 + ], + "ttft_s": [ + 0.02, + 0.04, + 0.06, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021619625389575958, + "decode_tps": 202.24917615461314, + "prefill_tps": 2497.730604806707, + "itl_mean_ms": 4.944395913066817, + "itl_p50_ms": 4.779167007654905, + "itl_p95_ms": 5.18479198217392, + "round_gap_p50_ms": 4.786790814250708, + "round_gap_p95_ms": 5.18479198217392, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04252487467601895, + "decode_tps": 205.65823600816472, + "prefill_tps": 1269.8450121583123, + "itl_mean_ms": 4.862435949126295, + "itl_p50_ms": 4.777375143021345, + "itl_p95_ms": 5.189749877899885, + "round_gap_p50_ms": 4.782957956194878, + "round_gap_p95_ms": 5.189749877899885, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06338662514463067, + "decode_tps": 209.18499878241553, + "prefill_tps": 851.9147355895191, + "itl_mean_ms": 4.780457517606955, + "itl_p50_ms": 4.777207970619202, + "itl_p95_ms": 5.18862484022975, + "round_gap_p50_ms": 4.783707670867443, + "round_gap_p95_ms": 5.18862484022975, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08441495895385742, + "decode_tps": 212.8034222193113, + "prefill_tps": 639.6970474097757, + "itl_mean_ms": 4.699172548876674, + "itl_p50_ms": 4.76783374324441, + "itl_p95_ms": 5.190291907638311, + "round_gap_p50_ms": 4.785833414644003, + "round_gap_p95_ms": 5.190291907638311, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.821615482169185, + "itl_p50_ms": 4.777207970619202, + "itl_p95_ms": 5.1852501928806305, + "round_gap_p50_ms": 4.786250181496143, + "round_gap_p95_ms": 5.1876250654459, + "round_gap_mean_ms": 4.918047791812569, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2558.7881896391027 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + } + ] + } + ] + }, + "vl-4-2-new.decode.json": [ + { + "label": "vl-4-2-new", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 301.7186871597126, + "decode_tps_all": [ + 301.64, + 302.86, + 301.72 + ], + "ttft_s_median": 0.021077708806842566, + "prefill_tps_median": 2514.505, + "itl_mean_ms_median": 3.314, + "itl_p50_ms_median": 3.366, + "itl_p95_ms_median": 3.865, + "round_gap_p50_ms_median": 3.38, + "round_gap_p95_ms_median": 3.865, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "sample": "Here's a complete implementation of a red-black tree in Python with detailed explanations and extensive unit tests:\n\n```python\nclass Node:\n def __init__(self" + }, + { + "label": "vl-4-2-new", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 299.99547068513067, + "decode_tps_all": [ + 300.0, + 299.93, + 300.62 + ], + "ttft_s_median": 0.021380832884460688, + "prefill_tps_median": 2525.627, + "itl_mean_ms_median": 3.333, + "itl_p50_ms_median": 3.4, + "itl_p95_ms_median": 3.856, + "round_gap_p50_ms_median": 3.411, + "round_gap_p95_ms_median": 3.856, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "sample": "# Matrix Multiplication on a GPU: A Detailed Tutorial\n\n## Introduction\n\nMatrix multiplication is a fundamental operation in linear algebra, and understanding ho" + } + ], + "vl-4-3-old.concurrent.json": { + "label": "vl-4-3-old", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 291.4, + "per_stream_tps": 291.4, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 291.441, + "prefill_tps": 2504.755, + "itl_mean_ms": 3.431, + "itl_p50_ms": 3.472, + "itl_p95_ms": 3.976, + "round_gap_p50_ms": 3.493, + "round_gap_p95_ms": 3.976, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2504.755, + "itl_mean_ms": 3.431, + "itl_p50_ms": 3.472, + "itl_p95_ms": 3.976, + "round_gap_p50_ms": 3.493, + "round_gap_p95_ms": 3.976, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 285.6590906606697, + "makespan_s": 0.8961731251329184, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 292.1869358436176, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8939619171433151, + "end_to_end_tps": 286.3656662445493, + "timing_valid": true, + "per_stream_tps": [ + 292.2 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021165167447179556, + "decode_tps": 292.1869358436176, + "prefill_tps": 2504.1143724597705, + "itl_mean_ms": 3.4224665011553204, + "itl_p50_ms": 3.471208270639181, + "itl_p95_ms": 3.969165962189436, + "round_gap_p50_ms": 3.490791656076908, + "round_gap_p95_ms": 3.969165962189436, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.4224665011553204, + "itl_p50_ms": 3.471208270639181, + "itl_p95_ms": 3.969165962189436, + "round_gap_p50_ms": 3.490791656076908, + "round_gap_p95_ms": 3.969165962189436, + "round_gap_mean_ms": 3.504935573472316, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2504.1143724597705 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 291.3455520146236, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8964601657353342, + "end_to_end_tps": 285.56762451348004, + "timing_valid": true, + "per_stream_tps": [ + 291.3 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021159749943763018, + "decode_tps": 291.3455520146236, + "prefill_tps": 2504.755497624494, + "itl_mean_ms": 3.432350324503347, + "itl_p50_ms": 3.4742923453450203, + "itl_p95_ms": 3.976666834205389, + "round_gap_p50_ms": 3.493417054414749, + "round_gap_p95_ms": 3.976666834205389, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.432350324503347, + "itl_p50_ms": 3.4742923453450203, + "itl_p95_ms": 3.976666834205389, + "round_gap_p50_ms": 3.493417054414749, + "round_gap_p95_ms": 3.976666834205389, + "round_gap_mean_ms": 3.5150575612383674, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2504.755497624494 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 291.44052049365393, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8961731251329184, + "end_to_end_tps": 285.6590906606697, + "timing_valid": true, + "per_stream_tps": [ + 291.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021159292198717594, + "decode_tps": 291.44052049365393, + "prefill_tps": 2504.8096837195803, + "itl_mean_ms": 3.431231862701037, + "itl_p50_ms": 3.4719579853117466, + "itl_p95_ms": 3.976458217948675, + "round_gap_p50_ms": 3.4942501224577427, + "round_gap_p95_ms": 3.976458217948675, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.431231862701037, + "itl_p50_ms": 3.4719579853117466, + "itl_p95_ms": 3.976458217948675, + "round_gap_p50_ms": 3.4942501224577427, + "round_gap_p95_ms": 3.976458217948675, + "round_gap_mean_ms": 3.513912148549255, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2504.8096837195803 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 650.6, + "per_stream_tps": 166.6, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.053, + "decode_tps": 166.657, + "prefill_tps": 1031.32, + "itl_mean_ms": 6.0, + "itl_p50_ms": 6.555, + "itl_p95_ms": 7.158, + "round_gap_p50_ms": 6.574, + "round_gap_p95_ms": 7.158, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2511.681, + "itl_mean_ms": 6.025, + "itl_p50_ms": 6.56, + "itl_p95_ms": 7.154, + "round_gap_p50_ms": 6.576, + "round_gap_p95_ms": 7.154, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 643.7595052221338, + "makespan_s": 1.5906561249867082, + "steady_tps": 675.6, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 647.2584323026626, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5978106670081615, + "end_to_end_tps": 640.8769331333857, + "timing_valid": true, + "steady_tps": 671.7942866908064, + "per_stream_tps": [ + 164.0, + 168.5, + 166.2, + 161.8 + ], + "ttft_s": [ + 0.04, + 0.08, + 0.06, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.04209416639059782, + "decode_tps": 163.98088570011979, + "prefill_tps": 1259.0818287789662, + "itl_mean_ms": 6.098271732894229, + "itl_p50_ms": 6.553750019520521, + "itl_p95_ms": 7.13970884680748, + "round_gap_p50_ms": 6.5711671486496925, + "round_gap_p95_ms": 7.13970884680748, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08346737502142787, + "decode_tps": 168.45655889746885, + "prefill_tps": 634.9786366995938, + "itl_mean_ms": 5.936248529264155, + "itl_p50_ms": 6.546749733388424, + "itl_p95_ms": 7.173416670411825, + "round_gap_p50_ms": 6.567875389009714, + "round_gap_p95_ms": 7.173416670411825, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06274262489750981, + "decode_tps": 166.19433890012562, + "prefill_tps": 844.7207952580178, + "itl_mean_ms": 6.017052124747458, + "itl_p50_ms": 6.56691612675786, + "itl_p95_ms": 7.154665887355804, + "round_gap_p50_ms": 6.578292232006788, + "round_gap_p95_ms": 7.154665887355804, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021403959020972252, + "decode_tps": 161.83015737876087, + "prefill_tps": 2476.1774187695364, + "itl_mean_ms": 6.179317972604551, + "itl_p50_ms": 6.560500245541334, + "itl_p95_ms": 7.168042007833719, + "round_gap_p50_ms": 6.581958383321762, + "round_gap_p95_ms": 7.168042007833719, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 6.057722589877598, + "itl_p50_ms": 6.559625267982483, + "itl_p95_ms": 7.154207676649094, + "round_gap_p50_ms": 6.576084066182375, + "round_gap_p95_ms": 7.154292427003384, + "round_gap_mean_ms": 6.17887704167515, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2537.7646971583404 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 650.5933261353097, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5906561249867082, + "end_to_end_tps": 643.7595052221338, + "timing_valid": true, + "steady_tps": 675.6372465548628, + "per_stream_tps": [ + 167.1, + 162.7, + 169.4, + 164.9 + ], + "ttft_s": [ + 0.06, + 0.02, + 0.08, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.0641712499782443, + "decode_tps": 167.11921753697195, + "prefill_tps": 825.9150323231721, + "itl_mean_ms": 5.983752286171212, + "itl_p50_ms": 6.573292426764965, + "itl_p95_ms": 7.194625213742256, + "round_gap_p50_ms": 6.58941688016057, + "round_gap_p95_ms": 7.194625213742256, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.02223208313807845, + "decode_tps": 162.67108034242156, + "prefill_tps": 2383.9421466188733, + "itl_mean_ms": 6.147374185350011, + "itl_p50_ms": 6.5733338706195354, + "itl_p95_ms": 7.137292064726353, + "round_gap_p50_ms": 6.5811253152787685, + "round_gap_p95_ms": 7.137292064726353, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08476912509649992, + "decode_tps": 169.41247280765444, + "prefill_tps": 625.2276396584911, + "itl_mean_ms": 5.9027531056427485, + "itl_p50_ms": 6.556875072419643, + "itl_p95_ms": 7.156000006943941, + "round_gap_p50_ms": 6.577374879270792, + "round_gap_p95_ms": 7.156000006943941, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04351687477901578, + "decode_tps": 164.87991081062285, + "prefill_tps": 1217.9183424623375, + "itl_mean_ms": 6.0650202628298135, + "itl_p50_ms": 6.588958203792572, + "itl_p95_ms": 7.1775829419493675, + "round_gap_p50_ms": 6.590750068426132, + "round_gap_p95_ms": 7.1775829419493675, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 6.024724959998446, + "itl_p50_ms": 6.572582758963108, + "itl_p95_ms": 7.1775829419493675, + "round_gap_p50_ms": 6.589583121240139, + "round_gap_p95_ms": 7.178707979619503, + "round_gap_mean_ms": 6.145219459198415, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2496.7870130022407 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 661.6437600577406, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5642437916249037, + "end_to_end_tps": 654.6294161323091, + "timing_valid": true, + "steady_tps": 687.4412350715758, + "per_stream_tps": [ + 165.5, + 167.7, + 170.0, + 172.3 + ], + "ttft_s": [ + 0.02, + 0.04, + 0.06, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.022287458181381226, + "decode_tps": 165.4563391963565, + "prefill_tps": 2378.019044104177, + "itl_mean_ms": 6.043890520345931, + "itl_p50_ms": 6.207417231053114, + "itl_p95_ms": 7.171833887696266, + "round_gap_p50_ms": 6.321541965007782, + "round_gap_p95_ms": 7.171833887696266, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04294708324596286, + "decode_tps": 167.69573020484546, + "prefill_tps": 1234.0768218522069, + "itl_mean_ms": 5.96318104687859, + "itl_p50_ms": 6.187541875988245, + "itl_p95_ms": 7.127125281840563, + "round_gap_p50_ms": 6.295083090662956, + "round_gap_p95_ms": 7.127125281840563, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06358920782804489, + "decode_tps": 170.00424067823434, + "prefill_tps": 833.4747641977275, + "itl_mean_ms": 5.8822062085656555, + "itl_p50_ms": 6.230874918401241, + "itl_p95_ms": 7.159708999097347, + "round_gap_p50_ms": 6.304000038653612, + "round_gap_p95_ms": 7.159708999097347, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08440562477335334, + "decode_tps": 172.34808465342752, + "prefill_tps": 627.9202380447515, + "itl_mean_ms": 5.802211274995523, + "itl_p50_ms": 6.127958185970783, + "itl_p95_ms": 7.108667399734259, + "round_gap_p50_ms": 6.2413327395915985, + "round_gap_p95_ms": 7.108667399734259, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.922872262696425, + "itl_p50_ms": 6.187541875988245, + "itl_p95_ms": 7.126749958842993, + "round_gap_p50_ms": 6.304000038653612, + "round_gap_p95_ms": 7.126958109438419, + "round_gap_mean_ms": 6.041329707950354, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2511.680952179006 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 289.8, + "per_stream_tps": 289.8, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.022, + "decode_tps": 289.758, + "prefill_tps": 2511.156, + "itl_mean_ms": 3.451, + "itl_p50_ms": 3.51, + "itl_p95_ms": 3.977, + "round_gap_p50_ms": 3.522, + "round_gap_p95_ms": 3.977, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2511.156, + "itl_mean_ms": 3.451, + "itl_p50_ms": 3.51, + "itl_p95_ms": 3.977, + "round_gap_p50_ms": 3.522, + "round_gap_p95_ms": 3.977, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 283.9391249155858, + "makespan_s": 0.9016017080284655, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 290.18977917247724, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.900297749787569, + "end_to_end_tps": 284.3503719301807, + "timing_valid": true, + "per_stream_tps": [ + 290.2 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021493833977729082, + "decode_tps": 290.18977917247724, + "prefill_tps": 2512.348427737569, + "itl_mean_ms": 3.446020748393208, + "itl_p50_ms": 3.5096672363579273, + "itl_p95_ms": 3.977375105023384, + "round_gap_p50_ms": 3.5217497497797012, + "round_gap_p95_ms": 3.977375105023384, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.446020748393208, + "itl_p50_ms": 3.5096672363579273, + "itl_p95_ms": 3.977375105023384, + "round_gap_p50_ms": 3.5217497497797012, + "round_gap_p95_ms": 3.977375105023384, + "round_gap_mean_ms": 3.529057392932804, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2512.348427737569 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 289.75774544125943, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9016017080284655, + "end_to_end_tps": 283.9391249155858, + "timing_valid": true, + "per_stream_tps": [ + 289.8 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021504042204469442, + "decode_tps": 289.75774544125943, + "prefill_tps": 2511.155785807402, + "itl_mean_ms": 3.4511588239932762, + "itl_p50_ms": 3.5057500936090946, + "itl_p95_ms": 3.9722081273794174, + "round_gap_p50_ms": 3.5222498700022697, + "round_gap_p95_ms": 3.9722081273794174, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.4511588239932762, + "itl_p50_ms": 3.5057500936090946, + "itl_p95_ms": 3.9722081273794174, + "round_gap_p50_ms": 3.5222498700022697, + "round_gap_p95_ms": 3.9722081273794174, + "round_gap_mean_ms": 3.5343192775834757, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2511.155785807402 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 289.5920749098784, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.9021250423975289, + "end_to_end_tps": 283.77440816812117, + "timing_valid": true, + "per_stream_tps": [ + 289.6 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021522625349462032, + "decode_tps": 289.5920749098784, + "prefill_tps": 2508.987594366584, + "itl_mean_ms": 3.4531331712416575, + "itl_p50_ms": 3.531333990395069, + "itl_p95_ms": 3.987375181168318, + "round_gap_p50_ms": 3.5352082923054695, + "round_gap_p95_ms": 3.987375181168318, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.4531331712416575, + "itl_p50_ms": 3.531333990395069, + "itl_p95_ms": 3.987375181168318, + "round_gap_p50_ms": 3.5352082923054695, + "round_gap_p95_ms": 3.987375181168318, + "round_gap_mean_ms": 3.536341199464348, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2508.987594366584 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 669.4, + "per_stream_tps": 171.8, + "ttft_s_max": 0.09, + "request": { + "ttft_s": 0.054, + "decode_tps": 171.802, + "prefill_tps": 1021.354, + "itl_mean_ms": 5.821, + "itl_p50_ms": 5.121, + "itl_p95_ms": 7.181, + "round_gap_p50_ms": 5.151, + "round_gap_p95_ms": 7.181, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2486.516, + "itl_mean_ms": 5.852, + "itl_p50_ms": 5.119, + "itl_p95_ms": 7.174, + "round_gap_p50_ms": 5.162, + "round_gap_p95_ms": 7.182, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 662.275141513796, + "makespan_s": 1.5461851665750146, + "steady_tps": 696.6, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 669.3586113036456, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5461851665750146, + "end_to_end_tps": 662.275141513796, + "timing_valid": true, + "steady_tps": 696.6387329576609, + "per_stream_tps": [ + 174.6, + 167.3, + 172.1, + 169.7 + ], + "ttft_s": [ + 0.08, + 0.02, + 0.06, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.08489754190668464, + "decode_tps": 174.580019486245, + "prefill_tps": 636.0608185729835, + "itl_mean_ms": 5.728032354119361, + "itl_p50_ms": 5.148875061422586, + "itl_p95_ms": 7.180917076766491, + "round_gap_p50_ms": 5.155333783477545, + "round_gap_p95_ms": 7.180917076766491, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021646750159561634, + "decode_tps": 167.34479589589208, + "prefill_tps": 2494.600787737532, + "itl_mean_ms": 5.975686274834123, + "itl_p50_ms": 5.155624821782112, + "itl_p95_ms": 7.18045886605978, + "round_gap_p50_ms": 5.162124987691641, + "round_gap_p95_ms": 7.18045886605978, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06374383298680186, + "decode_tps": 172.10114468635373, + "prefill_tps": 847.1407737777657, + "itl_mean_ms": 5.810536599407594, + "itl_p50_ms": 5.158124957233667, + "itl_p95_ms": 7.193000055849552, + "round_gap_p50_ms": 5.163166206330061, + "round_gap_p95_ms": 7.193000055849552, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04280741699039936, + "decode_tps": 169.68981645357854, + "prefill_tps": 1261.4636386986597, + "itl_mean_ms": 5.893105555179656, + "itl_p50_ms": 5.126667208969593, + "itl_p95_ms": 7.183500099927187, + "round_gap_p50_ms": 5.146208684891462, + "round_gap_p95_ms": 7.183500099927187, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.851840195885184, + "itl_p50_ms": 5.148458294570446, + "itl_p95_ms": 7.182208821177483, + "round_gap_p50_ms": 5.162083078175783, + "round_gap_p95_ms": 7.183500099927187, + "round_gap_mean_ms": 5.968876999802887, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2543.1723625796635 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 678.2656745715576, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5277338749729097, + "end_to_end_tps": 670.2738066982758, + "timing_valid": true, + "steady_tps": 706.7479220678672, + "per_stream_tps": [ + 169.6, + 177.1, + 172.1, + 174.6 + ], + "ttft_s": [ + 0.02, + 0.09, + 0.05, + 0.07 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.023241832852363586, + "decode_tps": 169.57617262954452, + "prefill_tps": 2323.3967967594454, + "itl_mean_ms": 5.897054901602221, + "itl_p50_ms": 5.10370871052146, + "itl_p95_ms": 7.166790775954723, + "round_gap_p50_ms": 5.112667102366686, + "round_gap_p95_ms": 7.166790775954723, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08718483289703727, + "decode_tps": 177.10845442193124, + "prefill_tps": 619.3737856190241, + "itl_mean_ms": 5.6462578438952855, + "itl_p50_ms": 5.0964998081326485, + "itl_p95_ms": 7.164791226387024, + "round_gap_p50_ms": 5.112749990075827, + "round_gap_p95_ms": 7.164791226387024, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.045166832860559225, + "decode_tps": 172.08318164543707, + "prefill_tps": 1195.567556545549, + "itl_mean_ms": 5.811143136930232, + "itl_p50_ms": 5.098708905279636, + "itl_p95_ms": 7.16879079118371, + "round_gap_p50_ms": 5.114250350743532, + "round_gap_p95_ms": 7.16879079118371, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06630945904180408, + "decode_tps": 174.5599321606924, + "prefill_tps": 814.3634525197422, + "itl_mean_ms": 5.7286915022368525, + "itl_p50_ms": 5.101458169519901, + "itl_p95_ms": 7.163957692682743, + "round_gap_p50_ms": 5.110332742333412, + "round_gap_p95_ms": 7.163957692682743, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.770786846166148, + "itl_p50_ms": 5.101000424474478, + "itl_p95_ms": 7.1654170751571655, + "round_gap_p50_ms": 5.113292019814253, + "round_gap_p95_ms": 7.165541872382164, + "round_gap_mean_ms": 5.886202583089471, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2472.8636148662317 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 666.6712051075426, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.5531384157948196, + "end_to_end_tps": 659.3102002927197, + "timing_valid": true, + "steady_tps": 694.2636810343557, + "per_stream_tps": [ + 169.1, + 174.0, + 171.5, + 166.7 + ], + "ttft_s": [ + 0.04, + 0.09, + 0.07, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.0440932079218328, + "decode_tps": 169.05916234117814, + "prefill_tps": 1224.6784152273449, + "itl_mean_ms": 5.9150890501983024, + "itl_p50_ms": 5.119000095874071, + "itl_p95_ms": 7.186375092715025, + "round_gap_p50_ms": 5.194374825805426, + "round_gap_p95_ms": 7.186375092715025, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.0867889579385519, + "decode_tps": 173.98173623116833, + "prefill_tps": 622.19896727223, + "itl_mean_ms": 5.747729742570835, + "itl_p50_ms": 5.100500304251909, + "itl_p95_ms": 7.182959001511335, + "round_gap_p50_ms": 5.13416575267911, + "round_gap_p95_ms": 7.182959001511335, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06569575006142259, + "decode_tps": 171.5215678732186, + "prefill_tps": 821.9709790893995, + "itl_mean_ms": 5.830170586705208, + "itl_p50_ms": 5.122541915625334, + "itl_p95_ms": 7.182791829109192, + "round_gap_p50_ms": 5.18829096108675, + "round_gap_p95_ms": 7.182791829109192, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.02255004085600376, + "decode_tps": 166.67575386950313, + "prefill_tps": 2394.6741535779943, + "itl_mean_ms": 5.999672878533602, + "itl_p50_ms": 5.147415678948164, + "itl_p95_ms": 7.165750022977591, + "round_gap_p50_ms": 5.16166677698493, + "round_gap_p95_ms": 7.165750022977591, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 5.873165564501987, + "itl_p50_ms": 5.119000095874071, + "itl_p95_ms": 7.173666730523109, + "round_gap_p50_ms": 5.174333229660988, + "round_gap_p95_ms": 7.18249985948205, + "round_gap_mean_ms": 5.9906288757920265, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2486.5158075364648 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true + } + } + ] + } + ] + } + ] + }, + "vl-4-3-old.decode.json": [ + { + "label": "vl-4-3-old", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 292.12128843744097, + "decode_tps_all": [ + 292.53, + 291.2, + 292.12 + ], + "ttft_s_median": 0.021182625088840723, + "prefill_tps_median": 2502.051, + "itl_mean_ms_median": 3.423, + "itl_p50_ms_median": 3.48, + "itl_p95_ms_median": 3.963, + "round_gap_p50_ms_median": 3.494, + "round_gap_p95_ms_median": 3.963, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "sample": "Here's a complete implementation of a red-black tree in Python with detailed explanations and extensive unit tests:\n\n```python\nclass Node:\n def __init__(self" + }, + { + "label": "vl-4-3-old", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 290.28747891593963, + "decode_tps_all": [ + 290.29, + 290.28, + 290.39 + ], + "ttft_s_median": 0.021479417104274035, + "prefill_tps_median": 2514.035, + "itl_mean_ms_median": 3.445, + "itl_p50_ms_median": 3.502, + "itl_p95_ms_median": 3.971, + "round_gap_p50_ms_median": 3.525, + "round_gap_p95_ms_median": 3.971, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "sample": "# Matrix Multiplication on a GPU: A Detailed Tutorial\n\n## Introduction\n\nMatrix multiplication is a fundamental operation in linear algebra, and understanding ho" + } + ] + }, + "limitations": [ + "Only4-bit checkpoint performance was compared in this ABBA experiment.", + "The decode client retains a12-digit token digest and full output-text digest; the concurrent client retains full256-token SHA-256 digests. All240 concurrent request digests match across legs for their respective prompts.", + "Each reported rate is the median of two per-leg medians; the raw three-repetition cells are retained.", + "These improvements do not by themselves establish parity with the standard server." + ] + } +} diff --git a/docs/receipts/lfm3/kv-pair.json b/docs/receipts/lfm3/kv-pair.json new file mode 100644 index 0000000000..ca3e6c47d4 --- /dev/null +++ b/docs/receipts/lfm3/kv-pair.json @@ -0,0 +1,25851 @@ +{ + "scope": "Reuse the qualified paired row helper for equal-geometry K/V; arithmetic and kernel sources unchanged. Matched 4-bit performance is recorded separately below; no standard-server gate pass.", + "source_commit": "021f5ce9f56c490762d842e1177dee9516c75837", + "host": "Apple M4 Pro 48 GiB, macOS 26.5.1", + "native_sha256": "e9a1488abb130af39c21ee08068fe490d5926271cebcd2e9d0f815711e3730b4", + "baseline_forward_sha256": "a05dfd241f67e05ab5ba8a0ea978c094b889818a463edab80f29848ceb25d4e3", + "candidate_forward_sha256": "8ef7afff7df80dda6d573c1c8c9b0079ecdfc0752fe35c827c4355d6cde549e2", + "fixture": "fidelity-prompts/reasoning-257.ids.json", + "tokens_npy_sha256": "81449804fdd1a104d6dc9607c27d5cea49237064e50c80a765f68d8eba57f596", + "all_six_logit_arrays_equal": true, + "equal_logit_bytes": 394752000, + "logit_arrays": [ + { + "model": "LFM3-1.7B-MLX-4bit", + "bytes": 65792000, + "sha256": "aaedf4b7a3db61d3ae5070f182e94faf81224ed5bd0db1145159aa1433ee0812" + }, + { + "model": "LFM3-1.7B-MLX-6bit", + "bytes": 65792000, + "sha256": "2710a9b6ca6033adbdcf00d80e787665998f2a3bceb5b0c2d6a113d9c3e18eac" + }, + { + "model": "LFM3-1.7B-MLX-8bit", + "bytes": 65792000, + "sha256": "68a9e475b459023bf3fbbae092b7cac795ddd64a4c6bad668481246cacc358e9" + }, + { + "model": "LFM3-VL-2B-MLX-4bit", + "bytes": 65792000, + "sha256": "118440ccd7fc07a502485a3266a2878ec815f316bcfa9dddd859e5d324fb8654" + }, + { + "model": "LFM3-VL-2B-MLX-6bit", + "bytes": 65792000, + "sha256": "9be9b95af0ff0c00697f8737ce9271ccac0e6dcd519b56d49f3666623ae6fd2c" + }, + { + "model": "LFM3-VL-2B-MLX-8bit", + "bytes": 65792000, + "sha256": "3d1b90a881f6adbd5df111c72f1b8bfe2f7a6d445e478a7be326e4627aa849b6" + } + ], + "operator_pairs": [ + { + "model": "LFM3-1.7B-MLX-4bit", + "rows": 1, + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.2.self_attn.v_proj", + "input_sha256": "dda2cad3821cfa4cf617e0326115349407f39fc0745dd9162f5956eabeea674c", + "result": "DUAL_PROJECT rows=1 k=2048 n=512 bits=4 group=64 equal_bytes=2048" + }, + { + "model": "LFM3-1.7B-MLX-4bit", + "rows": 3, + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.2.self_attn.v_proj", + "input_sha256": "badfc915b7fb3074e0473c7c12f57f7e9e12a5dc4b552e1750a97a55e2cfb13c", + "result": "DUAL_PROJECT rows=3 k=2048 n=512 bits=4 group=64 equal_bytes=6144" + }, + { + "model": "LFM3-1.7B-MLX-4bit", + "rows": 64, + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.2.self_attn.v_proj", + "input_sha256": "b63a3ce95ac63b4c6658f46c28ae63e60b1e7ec8df4523df47ad94dc6844f048", + "result": "DUAL_PROJECT rows=64 k=2048 n=512 bits=4 group=64 equal_bytes=131072" + }, + { + "model": "LFM3-1.7B-MLX-6bit", + "rows": 1, + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.2.self_attn.v_proj", + "input_sha256": "db10cfadbb4b600f83c224edbe51e63d1065f8e44f0c7b7b6d23f1df2b8cd0ab", + "result": "DUAL_PROJECT rows=1 k=2048 n=512 bits=6 group=64 equal_bytes=2048" + }, + { + "model": "LFM3-1.7B-MLX-6bit", + "rows": 3, + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.2.self_attn.v_proj", + "input_sha256": "40737eb9fd5bf11264c596149d31e5c6d07168e2d255ffae6f8c8605534c2ec9", + "result": "DUAL_PROJECT rows=3 k=2048 n=512 bits=6 group=64 equal_bytes=6144" + }, + { + "model": "LFM3-1.7B-MLX-6bit", + "rows": 64, + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.2.self_attn.v_proj", + "input_sha256": "cecf2508e4de61c6e75839e1c87764bcdae093d05d1f29d03831db1e476f01b3", + "result": "DUAL_PROJECT rows=64 k=2048 n=512 bits=6 group=64 equal_bytes=131072" + }, + { + "model": "LFM3-1.7B-MLX-8bit", + "rows": 1, + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.2.self_attn.v_proj", + "input_sha256": "db64b16362a24abdb2fbf3adba14d5fddd470ccbf017e5e1ee1532cf8eed7d90", + "result": "DUAL_PROJECT rows=1 k=2048 n=512 bits=8 group=64 equal_bytes=2048" + }, + { + "model": "LFM3-1.7B-MLX-8bit", + "rows": 3, + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.2.self_attn.v_proj", + "input_sha256": "4793dcbc5e1ca98f5ed831a1c7b83973a4654dbb2fff14fa745a32da567783e7", + "result": "DUAL_PROJECT rows=3 k=2048 n=512 bits=8 group=64 equal_bytes=6144" + }, + { + "model": "LFM3-1.7B-MLX-8bit", + "rows": 64, + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.2.self_attn.v_proj", + "input_sha256": "9c6a7b9422c6719539de6b1e5f47868f46309de3229daa915e6ec928d63d0d6d", + "result": "DUAL_PROJECT rows=64 k=2048 n=512 bits=8 group=64 equal_bytes=131072" + }, + { + "model": "LFM3-VL-2B-MLX-4bit", + "rows": 1, + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.2.self_attn.v_proj", + "input_sha256": "059da5133f34b6f00c1100a6a372585cc4d201daf53f04b49b6b996c355aeddb", + "result": "DUAL_PROJECT rows=1 k=2048 n=512 bits=4 group=32 equal_bytes=2048" + }, + { + "model": "LFM3-VL-2B-MLX-4bit", + "rows": 3, + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.2.self_attn.v_proj", + "input_sha256": "9c53356c2c353867d5548148f1db591c04a9c76931e9fe939fc3d1fe37d60623", + "result": "DUAL_PROJECT rows=3 k=2048 n=512 bits=4 group=32 equal_bytes=6144" + }, + { + "model": "LFM3-VL-2B-MLX-4bit", + "rows": 64, + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.2.self_attn.v_proj", + "input_sha256": "ac1a507d9a12b4da7985b104bb8518f5b8642c79ebb85920717e72cb503d7399", + "result": "DUAL_PROJECT rows=64 k=2048 n=512 bits=4 group=32 equal_bytes=131072" + }, + { + "model": "LFM3-VL-2B-MLX-6bit", + "rows": 1, + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.2.self_attn.v_proj", + "input_sha256": "5855f56f4481b466d18d5fc26643e557687259c5f9d4bf59356f7ac662a92c64", + "result": "DUAL_PROJECT rows=1 k=2048 n=512 bits=6 group=64 equal_bytes=2048" + }, + { + "model": "LFM3-VL-2B-MLX-6bit", + "rows": 3, + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.2.self_attn.v_proj", + "input_sha256": "45e9d76e6dbe6d2c91bf1e3865ef86e60236d667c38e602deb77aa833d5c17be", + "result": "DUAL_PROJECT rows=3 k=2048 n=512 bits=6 group=64 equal_bytes=6144" + }, + { + "model": "LFM3-VL-2B-MLX-6bit", + "rows": 64, + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.2.self_attn.v_proj", + "input_sha256": "35ec818b558d14a1b2243b8760ed0fd8bda97895272ea6ced3cc5ae6dcb5f0c9", + "result": "DUAL_PROJECT rows=64 k=2048 n=512 bits=6 group=64 equal_bytes=131072" + }, + { + "model": "LFM3-VL-2B-MLX-8bit", + "rows": 1, + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.2.self_attn.v_proj", + "input_sha256": "eeb398acf7512e1c70d698ca30854921368ce3eb128b6734656a4cc25bb77173", + "result": "DUAL_PROJECT rows=1 k=2048 n=512 bits=8 group=64 equal_bytes=2048" + }, + { + "model": "LFM3-VL-2B-MLX-8bit", + "rows": 3, + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.2.self_attn.v_proj", + "input_sha256": "d8388b6ed87375fbea31a1e09784ac1482a5cedfa678210be5877539637b0a97", + "result": "DUAL_PROJECT rows=3 k=2048 n=512 bits=8 group=64 equal_bytes=6144" + }, + { + "model": "LFM3-VL-2B-MLX-8bit", + "rows": 64, + "module_a": "language_model.model.layers.2.self_attn.k_proj", + "module_b": "language_model.model.layers.2.self_attn.v_proj", + "input_sha256": "4e0b08dc7cca474bcfb6d052a9cdd977b7ea8ae3eb08410eb1b6d453f3c34d30", + "result": "DUAL_PROJECT rows=64 k=2048 n=512 bits=8 group=64 equal_bytes=131072" + } + ], + "real_image8_logits": { + "values": 11520000, + "equal_to_retained_native_baseline": true, + "sha256": "db730509b6d7dcb3d6d134b59b6434dfb43ebc85cdd3fe351bc8cc4148b47642" + }, + "exactness_summary": "LFM3_EXACT mode=default passed=68 failed=0 skipped=0 prompt=5 windows=60 shared=2 snapshots=1 long=0 gpu_seconds=142.716", + "build_summaries": [ + "Build Summary: 69/69 steps succeeded", + "Build Summary: 724/724 steps succeeded; 982/1000 tests passed (18 skipped)" + ], + "source_sha256": { + "zig/src/families/lfm3/forward.zig": "2946616a0b565c8c9baea9d35da279d18b752a0f2a8cd3834dd9c2ab6d45d8c1", + "zig/src/families/lfm3/dual_projection.zig": "763f0b7f47d0b00642e6e911f6874dd2987716432ba7721b01811a0accf87cd3", + "zig/kernels/metal/lfm3/projectiondual.metal": "05a21d65ec7563d9fadecf49bd88269b17d351591f674a5359c5aacccbfc6601", + "zig/kernels/metal/core/row_projection.metal": "86963d6266a982f4cac7c85265048703841a9545c19f2d2c5271eb8e02a0879b" + }, + "serving": { + "scope": "Actual combined production binary, seeded sampling enabled, serial server ownership on M4. This is correctness evidence, not performance qualification.", + "models": [ + { + "model": "LFM3-1.7B-MLX-4bit", + "binary_sha256": "e9a1488abb130af39c21ee08068fe490d5926271cebcd2e9d0f815711e3730b4", + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "cmpl-536ffcbaac91a2e6ad4a11bd10dfc10f", + "first_content": " Then", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-720aaf0f358601123dc148faceff2650", + "first_content": "The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + } + } + ], + "speed_qualified": false + }, + { + "model": "LFM3-1.7B-MLX-6bit", + "binary_sha256": "e9a1488abb130af39c21ee08068fe490d5926271cebcd2e9d0f815711e3730b4", + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "cmpl-031a1ffd903210176798b8469aecab65", + "first_content": " Write", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-74a4f97e47436d535329d2b939ef1de7", + "first_content": "The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + } + } + ], + "speed_qualified": false + }, + { + "model": "LFM3-1.7B-MLX-8bit", + "binary_sha256": "e9a1488abb130af39c21ee08068fe490d5926271cebcd2e9d0f815711e3730b4", + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "cmpl-4eb602d2ae3cce3552d1f944faf3f7e5", + "first_content": " \n\n", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-043e8122da4c471c24e6564ffe234088", + "first_content": "The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + } + } + ], + "speed_qualified": false + }, + { + "model": "LFM3-VL-2B-MLX-4bit", + "binary_sha256": "e9a1488abb130af39c21ee08068fe490d5926271cebcd2e9d0f815711e3730b4", + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c", + "text": " \nThe story is a summary of the events that occur in the events that occur in the events that occur in the events" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + }, + { + "name": "red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead", + "text": "Red<|im_end|>\n<|im_end|>\n \"Red\" is a broad category; the specific shade appears to be a bright, saturated red" + } + }, + { + "name": "blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef", + "text": "Blue<|im_end|>\n<|im_end|>\n \"Blue\" is a broad category, but the image matches the specific shade \"Blue\" as" + } + }, + { + "name": "red_blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40", + "text": "Red, Blue<|im_end|>\n<|im_end|>\n<|im_end|>\n Name the solid color of the first image, then the second image." + } + }, + { + "name": "blue_red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf", + "text": "Blue, Red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to think about the visual characteristics of the images and the visual characteristics" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c", + "text": " \nThe story is a summary of the events that occur in the events that occur in the events that occur in the events" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c", + "text": " \nThe story is a summary of the events that occur in the events that occur in the events that occur in the events" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c", + "text": " \nThe story is a summary of the events that occur in the events that occur in the events that occur in the events" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + }, + { + "name": "red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead", + "text": "Red<|im_end|>\n<|im_end|>\n \"Red\" is a broad category; the specific shade appears to be a bright, saturated red" + } + }, + { + "name": "red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead", + "text": "Red<|im_end|>\n<|im_end|>\n \"Red\" is a broad category; the specific shade appears to be a bright, saturated red" + } + }, + { + "name": "blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef", + "text": "Blue<|im_end|>\n<|im_end|>\n \"Blue\" is a broad category, but the image matches the specific shade \"Blue\" as" + } + }, + { + "name": "blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef", + "text": "Blue<|im_end|>\n<|im_end|>\n \"Blue\" is a broad category, but the image matches the specific shade \"Blue\" as" + } + }, + { + "name": "red_blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40", + "text": "Red, Blue<|im_end|>\n<|im_end|>\n<|im_end|>\n Name the solid color of the first image, then the second image." + } + }, + { + "name": "red_blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40", + "text": "Red, Blue<|im_end|>\n<|im_end|>\n<|im_end|>\n Name the solid color of the first image, then the second image." + } + }, + { + "name": "blue_red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf", + "text": "Blue, Red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to think about the visual characteristics of the images and the visual characteristics" + } + }, + { + "name": "blue_red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf", + "text": "Blue, Red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to think about the visual characteristics of the images and the visual characteristics" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370", + "text": " A Python function that returns the larger of two integers can be written as: A can be returned as: A can be" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + ] + }, + { + "name": "red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red" + ], + "observed": [ + "red", + "red", + "red" + ], + "text": "Red<|im_end|>\n<|im_end|>\n \"Red\" is a broad category; the specific shade appears to be a bright, saturated red" + } + }, + { + "name": "blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue" + ], + "observed": [ + "blue", + "blue", + "blue" + ], + "text": "Blue<|im_end|>\n<|im_end|>\n \"Blue\" is a broad category, but the image matches the specific shade \"Blue\" as" + } + }, + { + "name": "red_blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red", + "blue" + ], + "observed": [ + "red", + "blue" + ], + "text": "Red, Blue<|im_end|>\n<|im_end|>\n<|im_end|>\n Name the solid color of the first image, then the second image." + } + }, + { + "name": "blue_red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue", + "red" + ], + "observed": [ + "blue", + "red" + ], + "text": "Blue, Red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to think about the visual characteristics of the images and the visual characteristics" + } + }, + { + "name": "images/concurrent_equals_solos", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead", + "text": "Red<|im_end|>\n<|im_end|>\n \"Red\" is a broad category; the specific shade appears to be a bright, saturated red" + }, + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef", + "text": "Blue<|im_end|>\n<|im_end|>\n \"Blue\" is a broad category, but the image matches the specific shade \"Blue\" as" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40", + "text": "Red, Blue<|im_end|>\n<|im_end|>\n<|im_end|>\n Name the solid color of the first image, then the second image." + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf", + "text": "Blue, Red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to think about the visual characteristics of the images and the visual characteristics" + } + ] + }, + { + "name": "images/corrupt_png_rejected", + "status": "passed", + "evidence": { + "error": "{\"error\": {\"message\": \"image 1 could not be decoded; send PNG, JPEG or WebP\", \"type\": \"invalid_request_error\"}}", + "status": 400, + "complete": false + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "cmpl-e31d67b690cff8251b2626b8ab41fa7f", + "first_content": " The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095", + "text": " The larger of two integers is the larger of the larger of the larger of the larger of the larger of the larger of" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-452cbb8f59081c5b97527ac005aa0d5b", + "first_content": "The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter blue wavelengths of sunlight more strongly than longer red wavelengths. This scattered" + } + }, + { + "name": "red/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-d50f511256612c298728051e97bb3a29", + "first_content": "Red", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 2, + "server_disconnections_after": 3, + "server_confirmed_cancel": true + } + }, + { + "name": "red/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead", + "text": "Red<|im_end|>\n<|im_end|>\n \"Red\" is a broad category; the specific shade appears to be a bright, saturated red" + } + }, + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead", + "text": "Red<|im_end|>\n<|im_end|>\n \"Red\" is a broad category; the specific shade appears to be a bright, saturated red" + } + } + ], + "speed_qualified": false + }, + { + "model": "LFM3-VL-2B-MLX-6bit", + "binary_sha256": "e9a1488abb130af39c21ee08068fe490d5926271cebcd2e9d0f815711e3730b4", + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942", + "text": " The story is about a child who watches birds near a certain place. The story is about a child who watches birds near" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + } + }, + { + "name": "red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is Red.<|im_end|>\n, ., and" + } + }, + { + "name": "blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0", + "text": "Blue<|im_end|>\n<|im_end|>\nfollowing the instruction to not mention the reasoning. Here is the final answer:\n\nBlue<|im_end|>\n" + } + }, + { + "name": "red_blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5", + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n named after the color of the sky on a clear day, but with a" + } + }, + { + "name": "blue_red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603", + "text": "Blue, red<|im_end|>\n<|im_end|>\n 2. The first image has a blue color, and the second image has" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942", + "text": " The story is about a child who watches birds near a certain place. The story is about a child who watches birds near" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942", + "text": " The story is about a child who watches birds near a certain place. The story is about a child who watches birds near" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942", + "text": " The story is about a child who watches birds near a certain place. The story is about a child who watches birds near" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + } + }, + { + "name": "red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is Red.<|im_end|>\n, ., and" + } + }, + { + "name": "red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is Red.<|im_end|>\n, ., and" + } + }, + { + "name": "blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0", + "text": "Blue<|im_end|>\n<|im_end|>\nfollowing the instruction to not mention the reasoning. Here is the final answer:\n\nBlue<|im_end|>\n" + } + }, + { + "name": "blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0", + "text": "Blue<|im_end|>\n<|im_end|>\nfollowing the instruction to not mention the reasoning. Here is the final answer:\n\nBlue<|im_end|>\n" + } + }, + { + "name": "red_blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5", + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n named after the color of the sky on a clear day, but with a" + } + }, + { + "name": "red_blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5", + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n named after the color of the sky on a clear day, but with a" + } + }, + { + "name": "blue_red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603", + "text": "Blue, red<|im_end|>\n<|im_end|>\n 2. The first image has a blue color, and the second image has" + } + }, + { + "name": "blue_red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603", + "text": "Blue, red<|im_end|>\n<|im_end|>\n 2. The first image has a blue color, and the second image has" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8", + "text": "The sky appears blue because molecules in the atmosphere scatter short blue wavelengths of sunlight more efficiently than longer red wavelengths. This scattering" + } + ] + }, + { + "name": "red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red" + ], + "observed": [ + "red", + "red" + ], + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is Red.<|im_end|>\n, ., and" + } + }, + { + "name": "blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue" + ], + "observed": [ + "blue", + "blue" + ], + "text": "Blue<|im_end|>\n<|im_end|>\nfollowing the instruction to not mention the reasoning. Here is the final answer:\n\nBlue<|im_end|>\n" + } + }, + { + "name": "red_blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red", + "blue" + ], + "observed": [ + "red", + "blue" + ], + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n named after the color of the sky on a clear day, but with a" + } + }, + { + "name": "blue_red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue", + "red" + ], + "observed": [ + "blue", + "red", + "blue" + ], + "text": "Blue, red<|im_end|>\n<|im_end|>\n 2. The first image has a blue color, and the second image has" + } + }, + { + "name": "images/concurrent_equals_solos", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is Red.<|im_end|>\n, ., and" + }, + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0", + "text": "Blue<|im_end|>\n<|im_end|>\nfollowing the instruction to not mention the reasoning. Here is the final answer:\n\nBlue<|im_end|>\n" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5", + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n named after the color of the sky on a clear day, but with a" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603", + "text": "Blue, red<|im_end|>\n<|im_end|>\n 2. The first image has a blue color, and the second image has" + } + ] + }, + { + "name": "images/corrupt_png_rejected", + "status": "passed", + "evidence": { + "error": "{\"error\": {\"message\": \"image 1 could not be decoded; send PNG, JPEG or WebP\", \"type\": \"invalid_request_error\"}}", + "status": 400, + "complete": false + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "cmpl-759d1d41e0f79f7620a766927d217518", + "first_content": " The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-ee48ddffd88da6dd42ceddef715d411d", + "first_content": "The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "red/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-758b732e65b0e64f0a2bff69b56677b7", + "first_content": "Red", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 2, + "server_disconnections_after": 3, + "server_confirmed_cancel": true + } + }, + { + "name": "red/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is Red.<|im_end|>\n, ., and" + } + }, + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is Red.<|im_end|>\n, ., and" + } + } + ], + "speed_qualified": false + }, + { + "model": "LFM3-VL-2B-MLX-8bit", + "binary_sha256": "e9a1488abb130af39c21ee08068fe490d5926271cebcd2e9d0f815711e3730b4", + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb", + "text": " The story is about a person who is always in a state of being in a state of a state of a state of" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + }, + { + "name": "red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is: Red<|im_end|>\nThe image contains a" + } + }, + { + "name": "blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6", + "text": "Blue<|im_end|>\nThe image contains a single solid color: Blue.<|im_end|>\n, I see a uniform blue hue throughout the" + } + }, + { + "name": "red_blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f", + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n colored with colored pencils pencil? No, solid color.<|im_end|>\n correction:" + } + }, + { + "name": "blue_red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005", + "text": "Blue, red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to rely on the visual information from the images provided. Rely" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb", + "text": " The story is about a person who is always in a state of being in a state of a state of a state of" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb", + "text": " The story is about a person who is always in a state of being in a state of a state of a state of" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb", + "text": " The story is about a person who is always in a state of being in a state of a state of a state of" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a", + "text": " The river flows past a quiet village. The river flows past a quiet village. The river flows past a quiet village." + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + }, + { + "name": "red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is: Red<|im_end|>\nThe image contains a" + } + }, + { + "name": "red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is: Red<|im_end|>\nThe image contains a" + } + }, + { + "name": "blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6", + "text": "Blue<|im_end|>\nThe image contains a single solid color: Blue.<|im_end|>\n, I see a uniform blue hue throughout the" + } + }, + { + "name": "blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6", + "text": "Blue<|im_end|>\nThe image contains a single solid color: Blue.<|im_end|>\n, I see a uniform blue hue throughout the" + } + }, + { + "name": "red_blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f", + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n colored with colored pencils pencil? No, solid color.<|im_end|>\n correction:" + } + }, + { + "name": "red_blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f", + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n colored with colored pencils pencil? No, solid color.<|im_end|>\n correction:" + } + }, + { + "name": "blue_red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005", + "text": "Blue, red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to rely on the visual information from the images provided. Rely" + } + }, + { + "name": "blue_red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005", + "text": "Blue, red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to rely on the visual information from the images provided. Rely" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa", + "text": " The larger of two integers: two integers. The larger of two integers: two integers. The larger of two integers:" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe", + "text": "The sky appears blue because air molecules scatter short blue wavelengths of sunlight more strongly than longer red wavelengths. As a result," + } + ] + }, + { + "name": "red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red" + ], + "observed": [ + "red", + "red" + ], + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is: Red<|im_end|>\nThe image contains a" + } + }, + { + "name": "blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue" + ], + "observed": [ + "blue", + "blue", + "blue" + ], + "text": "Blue<|im_end|>\nThe image contains a single solid color: Blue.<|im_end|>\n, I see a uniform blue hue throughout the" + } + }, + { + "name": "red_blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red", + "blue" + ], + "observed": [ + "red", + "blue" + ], + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n colored with colored pencils pencil? No, solid color.<|im_end|>\n correction:" + } + }, + { + "name": "blue_red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue", + "red" + ], + "observed": [ + "blue", + "red" + ], + "text": "Blue, red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to rely on the visual information from the images provided. Rely" + } + }, + { + "name": "images/concurrent_equals_solos", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is: Red<|im_end|>\nThe image contains a" + }, + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6", + "text": "Blue<|im_end|>\nThe image contains a single solid color: Blue.<|im_end|>\n, I see a uniform blue hue throughout the" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f", + "text": "Red, blue<|im_end|>\n<|im_end|>\n<|im_end|>\n colored with colored pencils pencil? No, solid color.<|im_end|>\n correction:" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005", + "text": "Blue, red<|im_end|>\n<|im_end|>\n<|im_end|>\n you have to rely on the visual information from the images provided. Rely" + } + ] + }, + { + "name": "images/corrupt_png_rejected", + "status": "passed", + "evidence": { + "error": "{\"error\": {\"message\": \"image 1 could not be decoded; send PNG, JPEG or WebP\", \"type\": \"invalid_request_error\"}}", + "status": 400, + "complete": false + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "cmpl-d36fae5ca96f49ae3cc6fdfeccd51d62", + "first_content": " The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258", + "text": " The larger of two integers is the larger of the two integers. The larger of two integers is the larger of the two" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-6f745a5056b68c43c09cb5874c07af3a", + "first_content": "The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength blue light more efficiently than longer-wavelength colors" + } + }, + { + "name": "red/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-5ba58499ac0a73d072b7d2080c489fe1", + "first_content": "Red", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 2, + "server_disconnections_after": 3, + "server_confirmed_cancel": true + } + }, + { + "name": "red/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is: Red<|im_end|>\nThe image contains a" + } + }, + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d", + "text": "Red<|im_end|>\n<|im_end|>\n one single solid color fills this image, and it is: Red<|im_end|>\nThe image contains a" + } + } + ], + "speed_qualified": false + } + ], + "counts": { + "passed": 303, + "failed": 0, + "skipped": 0 + } + }, + "extracted_text": { + "scope": "Actual extracted text-only binary after the same shared K/V dispatch change; combined-chain checks above use a different binary.", + "source_commit": "9a26db78d", + "binary_sha256": "034c5ad872c4226c4b4eae581f9bfa4065028dd4b03717613a28bb7d8e16bb3c", + "build_summary": "Build Summary: 721/721 steps succeeded; 980/998 tests passed (18 skipped)", + "python_tests": { + "passed": 39, + "failed": 0 + }, + "served": [ + { + "model": "LFM3-1.7B-MLX-4bit", + "binary_sha256": "034c5ad872c4226c4b4eae581f9bfa4065028dd4b03717613a28bb7d8e16bb3c", + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f", + "text": " \n``` \nYou must now use a `for` loop to use the function. \n``` \nYou must pass in" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677", + "text": "The sky appears blue because molecules in the atmosphere scatter short-wavelength light (blue) more efficiently than long-wavel" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "cmpl-f1cb7ed15ad5076f149ec5ac2669daab", + "first_content": " Then", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053", + "text": " Then, use that function to compare the numbers 10 and 20. Finally, compare the numbers 5 and " + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-91f6ab4e21eadd25ca5da09f379e45c7", + "first_content": "The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of light (like blue and green) more efficiently than longer" + } + } + ], + "speed_qualified": false + }, + { + "model": "LFM3-1.7B-MLX-6bit", + "binary_sha256": "034c5ad872c4226c4b4eae581f9bfa4065028dd4b03717613a28bb7d8e16bb3c", + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3", + "text": " \nWrite a Python function that takes a list of integers and returns a new list containing only the largest integer from each pair" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than longer" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "cmpl-ce0f09e3d60590d9ea0232e776c8407f", + "first_content": " Write", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0", + "text": " Write a Python function that takes two integers as arguments and returns the larger one. \n\n``` \n\nI need to write a" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-28c7bb3234998fed830dcd7d7592e8ad", + "first_content": "The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b", + "text": "The sky appears blue because molecules in the atmosphere scatter shorter (blue) wavelengths of sunlight more efficiently than longer (red)" + } + } + ], + "speed_qualified": false + }, + { + "model": "LFM3-1.7B-MLX-8bit", + "binary_sha256": "034c5ad872c4226c4b4eae581f9bfa4065028dd4b03717613a28bb7d8e16bb3c", + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e", + "text": " The river flows past a quiet village. Children watch birds near the bridge. The river flows past a quiet village. Children" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d", + "text": " \nWrite a function called `max_of_two` that takes two arguments and returns the larger of the two. \nEnsure" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093", + "text": "The sky appears blue because molecules in the Earth's atmosphere scatter shorter wavelengths of sunlight (blue and violet light) more efficiently" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "cmpl-0013b6b51147f9aab66a3ad69449f2b4", + "first_content": " \n\n", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f", + "text": " \n\n```python\ndef larger(a, b):\n if a > b:\n return a\n else:\n " + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "request_id": "chatcmpl-4cd62537f186c03c0445b4f5139bfe4b", + "first_content": "The", + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35", + "text": "The sky appears blue because molecules in Earth's atmosphere scatter shorter wavelengths of sunlight (like blue and violet) more efficiently than" + } + } + ], + "speed_qualified": false + } + ], + "counts": { + "passed": 120, + "failed": 0, + "skipped": 0 + }, + "vl_refusals": [ + { + "bits": 4, + "vl_refusal": true, + "exit_status": 1 + }, + { + "bits": 6, + "vl_refusal": true, + "exit_status": 1 + }, + { + "bits": 8, + "vl_refusal": true, + "exit_status": 1 + } + ] + }, + "performance": { + "scope": "M5 4-bit text/VL native old/new/new/old; three repetitions per code/prose prompt, 256 output tokens each. A small measured dispatch gain, not a standard-server gate pass.", + "hardware": "Apple M5 Ultra 256 GiB, macOS 27.0 build 26A428", + "summary": { + "baseline_sha256": "44220f39df174b2364172eac4b8705dbd5f5f47f3175c7ed300a6d47ae3bbcbc", + "candidate_sha256": "e9a1488abb130af39c21ee08068fe490d5926271cebcd2e9d0f815711e3730b4", + "order": [ + "old", + "new", + "new", + "old" + ], + "counts": { + "alone_equal": 240, + "alone_unequal": 0, + "alone_failed": 0, + "serial_equal": 32, + "serial_unequal": 0, + "serial_failed": 0 + }, + "full_hash_requests": 240, + "hash_groups": [ + { + "family": "text", + "prompt": "long-code-chat", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7" + }, + { + "family": "text", + "prompt": "long-prose-chat", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a" + }, + { + "family": "vl", + "prompt": "long-code-chat", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee" + }, + { + "family": "vl", + "prompt": "long-prose-chat", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa" + } + ], + "cells": [ + { + "family": "text", + "mode": "four_stream", + "prompt": "long-code-chat", + "old_run_medians": [ + 837.0, + 838.8 + ], + "new_run_medians": [ + 850.0, + 849.2 + ], + "old_median": 837.9, + "new_median": 849.6, + "rate_ratio": 1.0139634801288937 + }, + { + "family": "text", + "mode": "four_stream", + "prompt": "long-prose-chat", + "old_run_medians": [ + 857.0, + 858.1 + ], + "new_run_medians": [ + 869.5, + 868.0 + ], + "old_median": 857.55, + "new_median": 868.75, + "rate_ratio": 1.013060462946767 + }, + { + "family": "text", + "mode": "solo", + "prompt": "long-code-chat", + "old_run_medians": [ + 317.52504786591186, + 315.9518189760151 + ], + "new_run_medians": [ + 320.5081878043543, + 319.67577794253697 + ], + "old_median": 316.7384334209635, + "new_median": 320.09198287344566, + "rate_ratio": 1.0105877566427979 + }, + { + "family": "text", + "mode": "solo", + "prompt": "long-prose-chat", + "old_run_medians": [ + 333.35962421230994, + 331.1493532644477 + ], + "new_run_medians": [ + 335.87454621476815, + 335.0961656322742 + ], + "old_median": 332.2544887383788, + "new_median": 335.4853559235212, + "rate_ratio": 1.0097240738489657 + }, + { + "family": "vl", + "mode": "four_stream", + "prompt": "long-code-chat", + "old_run_medians": [ + 814.1, + 813.1 + ], + "new_run_medians": [ + 824.5, + 824.3 + ], + "old_median": 813.6, + "new_median": 824.4, + "rate_ratio": 1.0132743362831858 + }, + { + "family": "vl", + "mode": "four_stream", + "prompt": "long-prose-chat", + "old_run_medians": [ + 810.8, + 809.5 + ], + "new_run_medians": [ + 821.7, + 819.8 + ], + "old_median": 810.15, + "new_median": 820.75, + "rate_ratio": 1.0130839967907177 + }, + { + "family": "vl", + "mode": "solo", + "prompt": "long-code-chat", + "old_run_medians": [ + 301.8038551050975, + 302.2026871330548 + ], + "new_run_medians": [ + 305.9170505050233, + 305.175979302901 + ], + "old_median": 302.00327111907615, + "new_median": 305.54651490396213, + "rate_ratio": 1.0117324682337263 + }, + { + "family": "vl", + "mode": "solo", + "prompt": "long-prose-chat", + "old_run_medians": [ + 299.7549943842911, + 300.1215491541061 + ], + "new_run_medians": [ + 303.7439269888273, + 302.96962197420424 + ], + "old_median": 299.9382717691986, + "new_median": 303.3567744815158, + "rate_ratio": 1.0113973541694197 + } + ] + }, + "decode_client_scope": "Single-stream client stores a truncated token digest and full output-text SHA; full token SHA equality is established by concurrent request evidence.", + "clients": [ + { + "name": "text-4-0-old.decode.json", + "data": [ + { + "label": "text-4-0-old", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 317.52504786591186, + "decode_tps_all": [ + 317.79, + 317.53, + 317.21 + ], + "ttft_s_median": 0.020944457966834307, + "prefill_tps_median": 2530.502, + "itl_mean_ms_median": 3.149, + "itl_p50_ms_median": 3.212, + "itl_p95_ms_median": 3.674, + "round_gap_p50_ms_median": 3.217, + "round_gap_p95_ms_median": 3.674, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "sample": "Here is a complete Python implementation of a Red-Black Tree with insertion, deletion, searching, rotations, and extensive unit tests. I'll provide detailed exp" + }, + { + "label": "text-4-0-old", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 333.35962421230994, + "decode_tps_all": [ + 333.25, + 333.94, + 333.36 + ], + "ttft_s_median": 0.021223582793027163, + "prefill_tps_median": 2544.339, + "itl_mean_ms_median": 3.0, + "itl_p50_ms_median": 3.213, + "itl_p95_ms_median": 3.671, + "round_gap_p50_ms_median": 3.277, + "round_gap_p95_ms_median": 3.673, + "tokens_per_chunk_median": 1.103, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "sample": "We need to write a detailed 2000-word tutorial explaining how matrix multiplication runs on a GPU, covering memory layout, tiling, shared memory, synchro" + } + ] + }, + { + "name": "text-4-1-new.decode.json", + "data": [ + { + "label": "text-4-1-new", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 320.5081878043543, + "decode_tps_all": [ + 320.51, + 321.14, + 320.44 + ], + "ttft_s_median": 0.020921249873936176, + "prefill_tps_median": 2533.309, + "itl_mean_ms_median": 3.12, + "itl_p50_ms_median": 3.18, + "itl_p95_ms_median": 3.642, + "round_gap_p50_ms_median": 3.196, + "round_gap_p95_ms_median": 3.642, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "sample": "Here is a complete Python implementation of a Red-Black Tree with insertion, deletion, searching, rotations, and extensive unit tests. I'll provide detailed exp" + }, + { + "label": "text-4-1-new", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 335.87454621476815, + "decode_tps_all": [ + 335.82, + 335.87, + 336.81 + ], + "ttft_s_median": 0.021156500093638897, + "prefill_tps_median": 2552.407, + "itl_mean_ms_median": 2.977, + "itl_p50_ms_median": 3.181, + "itl_p95_ms_median": 3.648, + "round_gap_p50_ms_median": 3.245, + "round_gap_p95_ms_median": 3.651, + "tokens_per_chunk_median": 1.103, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "sample": "We need to write a detailed 2000-word tutorial explaining how matrix multiplication runs on a GPU, covering memory layout, tiling, shared memory, synchro" + } + ] + }, + { + "name": "text-4-2-new.decode.json", + "data": [ + { + "label": "text-4-2-new", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 319.67577794253697, + "decode_tps_all": [ + 319.68, + 320.04, + 319.56 + ], + "ttft_s_median": 0.020918624941259623, + "prefill_tps_median": 2533.627, + "itl_mean_ms_median": 3.128, + "itl_p50_ms_median": 3.177, + "itl_p95_ms_median": 3.648, + "round_gap_p50_ms_median": 3.185, + "round_gap_p95_ms_median": 3.648, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "sample": "Here is a complete Python implementation of a Red-Black Tree with insertion, deletion, searching, rotations, and extensive unit tests. I'll provide detailed exp" + }, + { + "label": "text-4-2-new", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 335.0961656322742, + "decode_tps_all": [ + 335.1, + 334.91, + 335.63 + ], + "ttft_s_median": 0.02119408268481493, + "prefill_tps_median": 2547.881, + "itl_mean_ms_median": 2.984, + "itl_p50_ms_median": 3.204, + "itl_p95_ms_median": 3.656, + "round_gap_p50_ms_median": 3.26, + "round_gap_p95_ms_median": 3.657, + "tokens_per_chunk_median": 1.103, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "sample": "We need to write a detailed 2000-word tutorial explaining how matrix multiplication runs on a GPU, covering memory layout, tiling, shared memory, synchro" + } + ] + }, + { + "name": "text-4-3-old.decode.json", + "data": [ + { + "label": "text-4-3-old", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 315.9518189760151, + "decode_tps_all": [ + 315.85, + 316.36, + 315.95 + ], + "ttft_s_median": 0.020964291878044605, + "prefill_tps_median": 2528.108, + "itl_mean_ms_median": 3.165, + "itl_p50_ms_median": 3.225, + "itl_p95_ms_median": 3.68, + "round_gap_p50_ms_median": 3.24, + "round_gap_p95_ms_median": 3.68, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "sample": "Here is a complete Python implementation of a Red-Black Tree with insertion, deletion, searching, rotations, and extensive unit tests. I'll provide detailed exp" + }, + { + "label": "text-4-3-old", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 331.1493532644477, + "decode_tps_all": [ + 332.16, + 331.15, + 330.78 + ], + "ttft_s_median": 0.021248084027320147, + "prefill_tps_median": 2541.406, + "itl_mean_ms_median": 3.02, + "itl_p50_ms_median": 3.241, + "itl_p95_ms_median": 3.692, + "round_gap_p50_ms_median": 3.298, + "round_gap_p95_ms_median": 3.694, + "tokens_per_chunk_median": 1.103, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "sample": "We need to write a detailed 2000-word tutorial explaining how matrix multiplication runs on a GPU, covering memory layout, tiling, shared memory, synchro" + } + ] + }, + { + "name": "vl-4-0-old.decode.json", + "data": [ + { + "label": "vl-4-0-old", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 301.8038551050975, + "decode_tps_all": [ + 301.86, + 300.45, + 301.8 + ], + "ttft_s_median": 0.02110629202798009, + "prefill_tps_median": 2511.1, + "itl_mean_ms_median": 3.313, + "itl_p50_ms_median": 3.365, + "itl_p95_ms_median": 3.865, + "round_gap_p50_ms_median": 3.376, + "round_gap_p95_ms_median": 3.865, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "sample": "Here's a complete implementation of a red-black tree in Python with detailed explanations and extensive unit tests:\n\n```python\nclass Node:\n def __init__(self" + }, + { + "label": "vl-4-0-old", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 299.7549943842911, + "decode_tps_all": [ + 299.75, + 299.6, + 299.76 + ], + "ttft_s_median": 0.021352417301386595, + "prefill_tps_median": 2528.988, + "itl_mean_ms_median": 3.336, + "itl_p50_ms_median": 3.408, + "itl_p95_ms_median": 3.863, + "round_gap_p50_ms_median": 3.42, + "round_gap_p95_ms_median": 3.863, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "sample": "# Matrix Multiplication on a GPU: A Detailed Tutorial\n\n## Introduction\n\nMatrix multiplication is a fundamental operation in linear algebra, and understanding ho" + } + ] + }, + { + "name": "vl-4-1-new.decode.json", + "data": [ + { + "label": "vl-4-1-new", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 305.9170505050233, + "decode_tps_all": [ + 305.92, + 305.66, + 306.2 + ], + "ttft_s_median": 0.02109841676428914, + "prefill_tps_median": 2512.037, + "itl_mean_ms_median": 3.269, + "itl_p50_ms_median": 3.32, + "itl_p95_ms_median": 3.816, + "round_gap_p50_ms_median": 3.332, + "round_gap_p95_ms_median": 3.816, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "sample": "Here's a complete implementation of a red-black tree in Python with detailed explanations and extensive unit tests:\n\n```python\nclass Node:\n def __init__(self" + }, + { + "label": "vl-4-1-new", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 303.7439269888273, + "decode_tps_all": [ + 303.97, + 303.27, + 303.74 + ], + "ttft_s_median": 0.021334875375032425, + "prefill_tps_median": 2531.067, + "itl_mean_ms_median": 3.292, + "itl_p50_ms_median": 3.357, + "itl_p95_ms_median": 3.823, + "round_gap_p50_ms_median": 3.375, + "round_gap_p95_ms_median": 3.823, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "sample": "# Matrix Multiplication on a GPU: A Detailed Tutorial\n\n## Introduction\n\nMatrix multiplication is a fundamental operation in linear algebra, and understanding ho" + } + ] + }, + { + "name": "vl-4-2-new.decode.json", + "data": [ + { + "label": "vl-4-2-new", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 305.175979302901, + "decode_tps_all": [ + 305.81, + 304.86, + 305.18 + ], + "ttft_s_median": 0.021077457815408707, + "prefill_tps_median": 2514.535, + "itl_mean_ms_median": 3.277, + "itl_p50_ms_median": 3.323, + "itl_p95_ms_median": 3.823, + "round_gap_p50_ms_median": 3.341, + "round_gap_p95_ms_median": 3.823, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "sample": "Here's a complete implementation of a red-black tree in Python with detailed explanations and extensive unit tests:\n\n```python\nclass Node:\n def __init__(self" + }, + { + "label": "vl-4-2-new", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 302.96962197420424, + "decode_tps_all": [ + 303.88, + 302.77, + 302.97 + ], + "ttft_s_median": 0.021318333223462105, + "prefill_tps_median": 2533.031, + "itl_mean_ms_median": 3.301, + "itl_p50_ms_median": 3.358, + "itl_p95_ms_median": 3.827, + "round_gap_p50_ms_median": 3.382, + "round_gap_p95_ms_median": 3.827, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "sample": "# Matrix Multiplication on a GPU: A Detailed Tutorial\n\n## Introduction\n\nMatrix multiplication is a fundamental operation in linear algebra, and understanding ho" + } + ] + }, + { + "name": "vl-4-3-old.decode.json", + "data": [ + { + "label": "vl-4-3-old", + "prompt": "long-code-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 302.2026871330548, + "decode_tps_all": [ + 301.97, + 302.2, + 302.27 + ], + "ttft_s_median": 0.021118917036801577, + "prefill_tps_median": 2509.598, + "itl_mean_ms_median": 3.309, + "itl_p50_ms_median": 3.363, + "itl_p95_ms_median": 3.854, + "round_gap_p50_ms_median": 3.381, + "round_gap_p95_ms_median": 3.854, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "sample": "Here's a complete implementation of a red-black tree in Python with detailed explanations and extensive unit tests:\n\n```python\nclass Node:\n def __init__(self" + }, + { + "label": "vl-4-3-old", + "prompt": "long-prose-chat", + "temperature": 0.0, + "tokens": 256, + "decode_tps_median": 300.1215491541061, + "decode_tps_all": [ + 300.12, + 300.61, + 299.6 + ], + "ttft_s_median": 0.021341833285987377, + "prefill_tps_median": 2530.242, + "itl_mean_ms_median": 3.332, + "itl_p50_ms_median": 3.402, + "itl_p95_ms_median": 3.865, + "round_gap_p50_ms_median": 3.413, + "round_gap_p95_ms_median": 3.865, + "tokens_per_chunk_median": 1.024, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "sample": "# Matrix Multiplication on a GPU: A Detailed Tutorial\n\n## Introduction\n\nMatrix multiplication is a fundamental operation in linear algebra, and understanding ho" + } + ] + }, + { + "name": "text-4-0-old.concurrent.json", + "data": { + "label": "text-4-0-old", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 316.7, + "per_stream_tps": 316.7, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 316.686, + "prefill_tps": 2531.409, + "itl_mean_ms": 3.158, + "itl_p50_ms": 3.206, + "itl_p95_ms": 3.687, + "round_gap_p50_ms": 3.224, + "round_gap_p95_ms": 3.687, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2531.409, + "itl_mean_ms": 3.158, + "itl_p50_ms": 3.206, + "itl_p95_ms": 3.687, + "round_gap_p50_ms": 3.224, + "round_gap_p95_ms": 3.687, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 309.85036713151896, + "makespan_s": 0.8262052498757839, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 318.45736759327224, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.821724290959537, + "end_to_end_tps": 311.54001751739116, + "timing_valid": true, + "per_stream_tps": [ + 318.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02093695802614093, + "decode_tps": 318.45736759327224, + "prefill_tps": 2531.4088099057476, + "itl_mean_ms": 3.1401377445196403, + "itl_p50_ms": 3.1895828433334827, + "itl_p95_ms": 3.6604171618819237, + "round_gap_p50_ms": 3.214708063751459, + "round_gap_p95_ms": 3.6604171618819237, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.1401377445196403, + "itl_p50_ms": 3.1895828433334827, + "itl_p95_ms": 3.6604171618819237, + "round_gap_p50_ms": 3.214708063751459, + "round_gap_p95_ms": 3.6604171618819237, + "round_gap_mean_ms": 3.2158037142671017, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2531.4088099057476 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 316.68601551004116, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8262052498757839, + "end_to_end_tps": 309.85036713151896, + "timing_valid": true, + "per_stream_tps": [ + 316.7 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.020893082953989506, + "decode_tps": 316.68601551004116, + "prefill_tps": 2536.7247196938793, + "itl_mean_ms": 3.157701796176387, + "itl_p50_ms": 3.2062078826129436, + "itl_p95_ms": 3.688374999910593, + "round_gap_p50_ms": 3.2235421240329742, + "round_gap_p95_ms": 3.688374999910593, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.157701796176387, + "itl_p50_ms": 3.2062078826129436, + "itl_p95_ms": 3.688374999910593, + "round_gap_p50_ms": 3.2235421240329742, + "round_gap_p95_ms": 3.688374999910593, + "round_gap_mean_ms": 3.2337909960842515, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2536.7247196938793 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 316.3907140132536, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8271391657181084, + "end_to_end_tps": 309.5005177003619, + "timing_valid": true, + "per_stream_tps": [ + 316.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02111987490206957, + "decode_tps": 316.3907140132536, + "prefill_tps": 2509.4845611422843, + "itl_mean_ms": 3.1606490194213155, + "itl_p50_ms": 3.221999853849411, + "itl_p95_ms": 3.686500247567892, + "round_gap_p50_ms": 3.2410421408712864, + "round_gap_p95_ms": 3.686500247567892, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.1606490194213155, + "itl_p50_ms": 3.221999853849411, + "itl_p95_ms": 3.686500247567892, + "round_gap_p50_ms": 3.2410421408712864, + "round_gap_p95_ms": 3.686500247567892, + "round_gap_mean_ms": 3.236809236756769, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2509.4845611422843 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 837.0, + "per_stream_tps": 214.8, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.052, + "decode_tps": 214.795, + "prefill_tps": 1060.106, + "itl_mean_ms": 4.656, + "itl_p50_ms": 4.599, + "itl_p95_ms": 5.011, + "round_gap_p50_ms": 4.607, + "round_gap_p95_ms": 5.011, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2561.366, + "itl_mean_ms": 4.657, + "itl_p50_ms": 4.6, + "itl_p95_ms": 5.012, + "round_gap_p50_ms": 4.611, + "round_gap_p95_ms": 5.012, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 825.7779541705285, + "makespan_s": 1.2400427921675146, + "steady_tps": 880.8, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 836.6646974329967, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2405616668984294, + "end_to_end_tps": 825.4325660086994, + "timing_valid": true, + "steady_tps": 880.4197266605183, + "per_stream_tps": [ + 216.5, + 209.2, + 220.3, + 212.8 + ], + "ttft_s": [ + 0.06, + 0.02, + 0.08, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.0622013327665627, + "decode_tps": 216.51455571212125, + "prefill_tps": 852.0717747143029, + "itl_mean_ms": 4.618627125141668, + "itl_p50_ms": 4.610165953636169, + "itl_p95_ms": 5.011333152651787, + "round_gap_p50_ms": 4.6154167503118515, + "round_gap_p95_ms": 5.011333152651787, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021178499795496464, + "decode_tps": 209.23735027490414, + "prefill_tps": 2502.537975389091, + "itl_mean_ms": 4.779261440111726, + "itl_p50_ms": 4.599499981850386, + "itl_p95_ms": 5.01812482252717, + "round_gap_p50_ms": 4.606292117387056, + "round_gap_p95_ms": 5.01812482252717, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08263254072517157, + "decode_tps": 220.2785875383358, + "prefill_tps": 641.3938084788323, + "itl_mean_ms": 4.539705884149846, + "itl_p50_ms": 4.6009160578250885, + "itl_p95_ms": 5.0104171968996525, + "round_gap_p50_ms": 4.605791997164488, + "round_gap_p95_ms": 5.0104171968996525, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.041769708041101694, + "decode_tps": 212.81500308957914, + "prefill_tps": 1268.8621128940526, + "itl_mean_ms": 4.69891683143728, + "itl_p50_ms": 4.599499981850386, + "itl_p95_ms": 5.012874957174063, + "round_gap_p50_ms": 4.604749847203493, + "round_gap_p95_ms": 5.012874957174063, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.65912782021013, + "itl_p50_ms": 4.6009160578250885, + "itl_p95_ms": 5.011915694922209, + "round_gap_p50_ms": 4.610165953636169, + "round_gap_p95_ms": 5.012250039726496, + "round_gap_mean_ms": 4.752310376614332, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2561.3660821542417 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 836.973695486923, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2400427921675146, + "end_to_end_tps": 825.7779541705285, + "timing_valid": true, + "steady_tps": 880.8018202197405, + "per_stream_tps": [ + 209.3, + 220.4, + 216.5, + 212.9 + ], + "ttft_s": [ + 0.02, + 0.08, + 0.06, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021297874860465527, + "decode_tps": 209.33403344537845, + "prefill_tps": 2488.5111940619945, + "itl_mean_ms": 4.777054086911912, + "itl_p50_ms": 4.603459034115076, + "itl_p95_ms": 5.013708956539631, + "round_gap_p50_ms": 4.606833215802908, + "round_gap_p95_ms": 5.013708956539631, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.0827073329128325, + "decode_tps": 220.3800561417299, + "prefill_tps": 640.8137964726554, + "itl_mean_ms": 4.537615687677674, + "itl_p50_ms": 4.594541620463133, + "itl_p95_ms": 5.010624881833792, + "round_gap_p50_ms": 4.61137481033802, + "round_gap_p95_ms": 5.010624881833792, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06217883387580514, + "decode_tps": 216.5189143431996, + "prefill_tps": 852.3800897562863, + "itl_mean_ms": 4.618534149930758, + "itl_p50_ms": 4.5989579521119595, + "itl_p95_ms": 5.021166987717152, + "round_gap_p50_ms": 4.616917110979557, + "round_gap_p95_ms": 5.021166987717152, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04183829203248024, + "decode_tps": 212.91387348339353, + "prefill_tps": 1266.7821133533514, + "itl_mean_ms": 4.6967348047331265, + "itl_p50_ms": 4.58958325907588, + "itl_p95_ms": 5.014334339648485, + "round_gap_p50_ms": 4.602416884154081, + "round_gap_p95_ms": 5.014334339648485, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.657484682313368, + "itl_p50_ms": 4.599875304847956, + "itl_p95_ms": 5.013708956539631, + "round_gap_p50_ms": 4.613957833498716, + "round_gap_p95_ms": 5.013791844248772, + "round_gap_mean_ms": 4.750634375959635, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2560.776931113704 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 837.6501571749706, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.23892558272928, + "end_to_end_tps": 826.5226049688863, + "timing_valid": true, + "steady_tps": 881.5174882630834, + "per_stream_tps": [ + 216.8, + 213.1, + 220.5, + 209.5 + ], + "ttft_s": [ + 0.06, + 0.04, + 0.08, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.062102291733026505, + "decode_tps": 216.78246235845953, + "prefill_tps": 853.4306628786481, + "itl_mean_ms": 4.612919279173309, + "itl_p50_ms": 4.600582644343376, + "itl_p95_ms": 5.009166896343231, + "round_gap_p50_ms": 4.6147918328642845, + "round_gap_p95_ms": 5.009166896343231, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04159779101610184, + "decode_tps": 213.07514758316057, + "prefill_tps": 1274.1061173052326, + "itl_mean_ms": 4.693179900812752, + "itl_p50_ms": 4.602624569088221, + "itl_p95_ms": 5.002625286579132, + "round_gap_p50_ms": 4.611165728420019, + "round_gap_p95_ms": 5.002625286579132, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08252304187044501, + "decode_tps": 220.53966638550705, + "prefill_tps": 642.2448663878148, + "itl_mean_ms": 4.534331698189764, + "itl_p50_ms": 4.597499966621399, + "itl_p95_ms": 4.992665722966194, + "round_gap_p50_ms": 4.603874869644642, + "round_gap_p95_ms": 4.992665722966194, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021213332656770945, + "decode_tps": 209.49604492939613, + "prefill_tps": 2498.4287409024, + "itl_mean_ms": 4.773359804176817, + "itl_p50_ms": 4.594209138303995, + "itl_p95_ms": 5.003834143280983, + "round_gap_p50_ms": 4.602459259331226, + "round_gap_p95_ms": 5.003834143280983, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.65344767058816, + "itl_p50_ms": 4.597875289618969, + "itl_p95_ms": 5.001333076506853, + "round_gap_p50_ms": 4.611165728420019, + "round_gap_p95_ms": 5.0017498433589935, + "round_gap_mean_ms": 4.7465166239999235, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2565.0061256262993 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 332.2, + "per_stream_tps": 332.2, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 332.195, + "prefill_tps": 2543.426, + "itl_mean_ms": 3.01, + "itl_p50_ms": 3.229, + "itl_p95_ms": 3.684, + "round_gap_p50_ms": 3.294, + "round_gap_p95_ms": 3.684, + "tokens_per_chunk": 1.103 + }, + "pooled": { + "aggregate_prefill_tps": 2543.426, + "itl_mean_ms": 3.01, + "itl_p50_ms": 3.229, + "itl_p95_ms": 3.684, + "round_gap_p50_ms": 3.294, + "round_gap_p95_ms": 3.684, + "tokens_per_chunk": 1.103 + }, + "timing_valid": true, + "end_to_end_tps": 324.4955480313207, + "makespan_s": 0.788916832767427, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 332.19451750565287, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.788916832767427, + "end_to_end_tps": 324.4955480313207, + "timing_valid": true, + "per_stream_tps": [ + 332.2 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02123924996703863, + "decode_tps": 332.19451750565287, + "prefill_tps": 2542.4626615253856, + "itl_mean_ms": 3.0102844788309406, + "itl_p50_ms": 3.230833914130926, + "itl_p95_ms": 3.684000112116337, + "round_gap_p50_ms": 3.281624987721443, + "round_gap_p95_ms": 3.684499766677618, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.0102844788309406, + "itl_p50_ms": 3.230833914130926, + "itl_p95_ms": 3.684000112116337, + "round_gap_p50_ms": 3.281624987721443, + "round_gap_p95_ms": 3.684499766677618, + "round_gap_mean_ms": 3.323041307800389, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2542.4626615253856 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 331.5032761726438, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7905069156549871, + "end_to_end_tps": 323.8428341741794, + "timing_valid": true, + "per_stream_tps": [ + 331.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02121729077771306, + "decode_tps": 331.5032761726438, + "prefill_tps": 2545.094025704844, + "itl_mean_ms": 3.0165614395895424, + "itl_p50_ms": 3.2286252826452255, + "itl_p95_ms": 3.6967922933399677, + "round_gap_p50_ms": 3.2978751696646214, + "round_gap_p95_ms": 3.6995839327573776, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.0165614395895424, + "itl_p50_ms": 3.2286252826452255, + "itl_p95_ms": 3.6967922933399677, + "round_gap_p50_ms": 3.2978751696646214, + "round_gap_p95_ms": 3.6995839327573776, + "round_gap_mean_ms": 3.3299704203261182, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2545.094025704844 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 332.2522829461861, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7887705420143902, + "end_to_end_tps": 324.5557311841516, + "timing_valid": true, + "per_stream_tps": [ + 332.3 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.0212312089279294, + "decode_tps": 332.2522829461861, + "prefill_tps": 2543.4255855757533, + "itl_mean_ms": 3.009761110240338, + "itl_p50_ms": 3.2197916880249977, + "itl_p95_ms": 3.6802920512855053, + "round_gap_p50_ms": 3.293875139206648, + "round_gap_p95_ms": 3.6844168789684772, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.009761110240338, + "itl_p50_ms": 3.2197916880249977, + "itl_p95_ms": 3.6802920512855053, + "round_gap_p50_ms": 3.293875139206648, + "round_gap_p95_ms": 3.6844168789684772, + "round_gap_mean_ms": 3.322463563252321, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2543.4255855757533 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 857.0, + "per_stream_tps": 220.0, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.053, + "decode_tps": 220.025, + "prefill_tps": 1066.664, + "itl_mean_ms": 4.545, + "itl_p50_ms": 4.632, + "itl_p95_ms": 5.031, + "round_gap_p50_ms": 4.671, + "round_gap_p95_ms": 5.045, + "tokens_per_chunk": 1.08 + }, + "pooled": { + "aggregate_prefill_tps": 2575.865, + "itl_mean_ms": 4.545, + "itl_p50_ms": 4.633, + "itl_p95_ms": 5.029, + "round_gap_p50_ms": 4.675, + "round_gap_p95_ms": 5.044, + "tokens_per_chunk": 1.08 + }, + "timing_valid": true, + "end_to_end_tps": 844.7881254437754, + "makespan_s": 1.2121382500045002, + "steady_tps": 903.4, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 856.9961235338005, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2121382500045002, + "end_to_end_tps": 844.7881254437754, + "timing_valid": true, + "steady_tps": 903.4092453845401, + "per_stream_tps": [ + 218.1, + 222.0, + 226.1, + 214.3 + ], + "ttft_s": [ + 0.04, + 0.06, + 0.08, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.04226362472400069, + "decode_tps": 218.06513585847028, + "prefill_tps": 1277.6944796534324, + "itl_mean_ms": 4.585785783973395, + "itl_p50_ms": 4.641958046704531, + "itl_p95_ms": 5.045542027801275, + "round_gap_p50_ms": 4.66883322224021, + "round_gap_p95_ms": 5.0473748706281185, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06298262486234307, + "decode_tps": 222.0103127264939, + "prefill_tps": 857.3793187887009, + "itl_mean_ms": 4.504295263220282, + "itl_p50_ms": 4.627209156751633, + "itl_p95_ms": 5.044999998062849, + "round_gap_p50_ms": 4.676166921854019, + "round_gap_p95_ms": 5.048458930104971, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08372183283790946, + "decode_tps": 226.0979383136288, + "prefill_tps": 644.9930462529084, + "itl_mean_ms": 4.422862090024293, + "itl_p50_ms": 4.632541909813881, + "itl_p95_ms": 5.033041816204786, + "round_gap_p50_ms": 4.672207869589329, + "round_gap_p95_ms": 5.0362092442810535, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.02148141572251916, + "decode_tps": 214.25481388040026, + "prefill_tps": 2513.8007986778694, + "itl_mean_ms": 4.667339705880366, + "itl_p50_ms": 4.6317907981574535, + "itl_p95_ms": 5.0559998489916325, + "round_gap_p50_ms": 4.6761659905314445, + "round_gap_p95_ms": 5.056875292211771, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.545070710774584, + "itl_p50_ms": 4.632541909813881, + "itl_p95_ms": 5.038458853960037, + "round_gap_p50_ms": 4.6755000948905945, + "round_gap_p95_ms": 5.044999998062849, + "round_gap_mean_ms": 4.9109874205403345, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2575.864799176197 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 857.2202949858439, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2118793330155313, + "end_to_end_tps": 844.968613708405, + "timing_valid": true, + "steady_tps": 903.7019860997952, + "per_stream_tps": [ + 222.1, + 218.1, + 226.2, + 214.3 + ], + "ttft_s": [ + 0.06, + 0.04, + 0.08, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.06315112486481667, + "decode_tps": 222.08474564385705, + "prefill_tps": 855.0916569672218, + "itl_mean_ms": 4.502785624023161, + "itl_p50_ms": 4.639958031475544, + "itl_p95_ms": 5.022000055760145, + "round_gap_p50_ms": 4.66995919123292, + "round_gap_p95_ms": 5.032083950936794, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.042321457993239164, + "decode_tps": 218.13600567640844, + "prefill_tps": 1275.9484800506277, + "itl_mean_ms": 4.584295916206696, + "itl_p50_ms": 4.637417383491993, + "itl_p95_ms": 5.010792054235935, + "round_gap_p50_ms": 4.679250065237284, + "round_gap_p95_ms": 5.0110407173633575, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08381904195994139, + "decode_tps": 226.170707874834, + "prefill_tps": 644.2450156589425, + "itl_mean_ms": 4.421439051043754, + "itl_p50_ms": 4.630833398550749, + "itl_p95_ms": 5.005540791898966, + "round_gap_p50_ms": 4.6685002744197845, + "round_gap_p95_ms": 5.0096670165658, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021529582794755697, + "decode_tps": 214.30750518325107, + "prefill_tps": 2508.1767963080842, + "itl_mean_ms": 4.666192157595765, + "itl_p50_ms": 4.650665912777185, + "itl_p95_ms": 5.01291686668992, + "round_gap_p50_ms": 4.6751247718930244, + "round_gap_p95_ms": 5.026000086218119, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.543678187217345, + "itl_p50_ms": 4.6334583312273026, + "itl_p95_ms": 5.0097499042749405, + "round_gap_p50_ms": 4.6751247718930244, + "round_gap_p95_ms": 5.01291686668992, + "round_gap_mean_ms": 4.90948278703569, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2572.803031664288 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 856.6870671284013, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2125837500207126, + "end_to_end_tps": 844.4777525531813, + "timing_valid": true, + "steady_tps": 903.004968486288, + "per_stream_tps": [ + 214.2, + 218.0, + 226.0, + 221.9 + ], + "ttft_s": [ + 0.02, + 0.04, + 0.08, + 0.06 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021424791775643826, + "decode_tps": 214.17976439310826, + "prefill_tps": 2520.4445655984573, + "itl_mean_ms": 4.668975161278949, + "itl_p50_ms": 4.629000090062618, + "itl_p95_ms": 5.03670796751976, + "round_gap_p50_ms": 4.665042273700237, + "round_gap_p95_ms": 5.044208373874426, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04219312500208616, + "decode_tps": 217.9841325474184, + "prefill_tps": 1279.8293560225763, + "itl_mean_ms": 4.587489870541236, + "itl_p50_ms": 4.6289172023534775, + "itl_p95_ms": 5.041416734457016, + "round_gap_p50_ms": 4.66491561383009, + "round_gap_p95_ms": 5.045375321060419, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08362433314323425, + "decode_tps": 225.99612773494107, + "prefill_tps": 645.7450597245086, + "itl_mean_ms": 4.424854576149407, + "itl_p50_ms": 4.630874842405319, + "itl_p95_ms": 5.0274161621928215, + "round_gap_p50_ms": 4.670500289648771, + "round_gap_p95_ms": 5.046166945248842, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.0630067503079772, + "decode_tps": 221.9134691222159, + "prefill_tps": 857.0510260574911, + "itl_mean_ms": 4.50626094916872, + "itl_p50_ms": 4.6385000459849834, + "itl_p95_ms": 5.028749816119671, + "round_gap_p50_ms": 4.680875223129988, + "round_gap_p95_ms": 5.04683330655098, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.546895139284578, + "itl_p50_ms": 4.630125127732754, + "itl_p95_ms": 5.028749816119671, + "round_gap_p50_ms": 4.671792034059763, + "round_gap_p95_ms": 5.044208373874426, + "round_gap_mean_ms": 4.912958731006642, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2578.913301774272 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + } + ] + } + ] + } + }, + { + "name": "text-4-1-new.concurrent.json", + "data": { + "label": "text-4-1-new", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 321.0, + "per_stream_tps": 321.0, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 320.978, + "prefill_tps": 2533.435, + "itl_mean_ms": 3.115, + "itl_p50_ms": 3.167, + "itl_p95_ms": 3.636, + "round_gap_p50_ms": 3.19, + "round_gap_p95_ms": 3.636, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2533.435, + "itl_mean_ms": 3.115, + "itl_p50_ms": 3.167, + "itl_p95_ms": 3.636, + "round_gap_p50_ms": 3.19, + "round_gap_p95_ms": 3.636, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 313.9495706884247, + "makespan_s": 0.8154175826348364, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 321.74083255144575, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8135659587569535, + "end_to_end_tps": 314.6640997506116, + "timing_valid": true, + "per_stream_tps": [ + 321.7 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.020920209120959044, + "decode_tps": 321.74083255144575, + "prefill_tps": 2533.435478276439, + "itl_mean_ms": 3.1080916651762, + "itl_p50_ms": 3.1568328849971294, + "itl_p95_ms": 3.6357082426548004, + "round_gap_p50_ms": 3.176500089466572, + "round_gap_p95_ms": 3.6357082426548004, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.1080916651762, + "itl_p50_ms": 3.1568328849971294, + "itl_p95_ms": 3.6357082426548004, + "round_gap_p50_ms": 3.176500089466572, + "round_gap_p95_ms": 3.6357082426548004, + "round_gap_mean_ms": 3.1829854402406865, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2533.435478276439 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 320.7423109636263, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8160214577801526, + "end_to_end_tps": 313.7172405936535, + "timing_valid": true, + "per_stream_tps": [ + 320.7 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.020924165844917297, + "decode_tps": 320.7423109636263, + "prefill_tps": 2532.95640996242, + "itl_mean_ms": 3.1177676465435353, + "itl_p50_ms": 3.173290751874447, + "itl_p95_ms": 3.633917309343815, + "round_gap_p50_ms": 3.196875099092722, + "round_gap_p95_ms": 3.633917309343815, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.1177676465435353, + "itl_p50_ms": 3.173290751874447, + "itl_p95_ms": 3.633917309343815, + "round_gap_p50_ms": 3.196875099092722, + "round_gap_p95_ms": 3.633917309343815, + "round_gap_mean_ms": 3.1928945777855486, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2532.95640996242 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 320.97817353604785, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8154175826348364, + "end_to_end_tps": 313.9495706884247, + "timing_valid": true, + "per_stream_tps": [ + 321.0 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02090858295559883, + "decode_tps": 320.97817353604785, + "prefill_tps": 2534.8441887501435, + "itl_mean_ms": 3.1154766350108036, + "itl_p50_ms": 3.1667500734329224, + "itl_p95_ms": 3.6355420015752316, + "round_gap_p50_ms": 3.1900829635560513, + "round_gap_p95_ms": 3.6355420015752316, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.1154766350108036, + "itl_p50_ms": 3.1667500734329224, + "itl_p95_ms": 3.6355420015752316, + "round_gap_p50_ms": 3.1900829635560513, + "round_gap_p95_ms": 3.6355420015752316, + "round_gap_mean_ms": 3.190548361155642, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2534.8441887501435 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 850.0, + "per_stream_tps": 218.1, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.052, + "decode_tps": 218.067, + "prefill_tps": 1062.4, + "itl_mean_ms": 4.586, + "itl_p50_ms": 4.527, + "itl_p95_ms": 4.94, + "round_gap_p50_ms": 4.54, + "round_gap_p95_ms": 4.94, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2565.58, + "itl_mean_ms": 4.585, + "itl_p50_ms": 4.527, + "itl_p95_ms": 4.939, + "round_gap_p50_ms": 4.541, + "round_gap_p95_ms": 4.941, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 838.0831586784976, + "makespan_s": 1.2218357920646667, + "steady_tps": 894.9, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 849.4010838230959, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2222774159163237, + "end_to_end_tps": 837.78034893357, + "timing_valid": true, + "steady_tps": 894.3515647154059, + "per_stream_tps": [ + 219.9, + 212.4, + 216.1, + 223.8 + ], + "ttft_s": [ + 0.06, + 0.02, + 0.04, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.06209208304062486, + "decode_tps": 219.88071622888685, + "prefill_tps": 853.5709772423612, + "itl_mean_ms": 4.547920423176358, + "itl_p50_ms": 4.532875027507544, + "itl_p95_ms": 4.942124709486961, + "round_gap_p50_ms": 4.549916833639145, + "round_gap_p95_ms": 4.942124709486961, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021095499861985445, + "decode_tps": 212.39280078713375, + "prefill_tps": 2512.3841741957094, + "itl_mean_ms": 4.708257512938158, + "itl_p50_ms": 4.53341705724597, + "itl_p95_ms": 4.943083971738815, + "round_gap_p50_ms": 4.553915932774544, + "round_gap_p95_ms": 4.943083971738815, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04151687491685152, + "decode_tps": 216.06979572919553, + "prefill_tps": 1276.5893412292342, + "itl_mean_ms": 4.628134148158863, + "itl_p50_ms": 4.530792124569416, + "itl_p95_ms": 4.944541025906801, + "round_gap_p50_ms": 4.542083013802767, + "round_gap_p95_ms": 4.944541025906801, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08259191596880555, + "decode_tps": 223.7972459966636, + "prefill_tps": 641.709292953293, + "itl_mean_ms": 4.468330231440418, + "itl_p50_ms": 4.531083628535271, + "itl_p95_ms": 4.939415957778692, + "round_gap_p50_ms": 4.539916291832924, + "round_gap_p95_ms": 4.939415957778692, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.588160578928449, + "itl_p50_ms": 4.531083628535271, + "itl_p95_ms": 4.941374994814396, + "round_gap_p50_ms": 4.542957991361618, + "round_gap_p95_ms": 4.941457882523537, + "round_gap_mean_ms": 4.679923790507019, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2566.837171813172 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 849.9675362701744, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2218357920646667, + "end_to_end_tps": 838.0831586784976, + "timing_valid": true, + "steady_tps": 894.8520724955473, + "per_stream_tps": [ + 212.5, + 216.2, + 224.0, + 220.0 + ], + "ttft_s": [ + 0.02, + 0.04, + 0.08, + 0.06 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021247500088065863, + "decode_tps": 212.5017340504427, + "prefill_tps": 2494.411096850337, + "itl_mean_ms": 4.7058439521374655, + "itl_p50_ms": 4.530624952167273, + "itl_p95_ms": 4.934499971568584, + "round_gap_p50_ms": 4.54079220071435, + "round_gap_p95_ms": 4.934499971568584, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.041716750245541334, + "decode_tps": 216.18849095249178, + "prefill_tps": 1270.4728841064175, + "itl_mean_ms": 4.625593136776896, + "itl_p50_ms": 4.520708229392767, + "itl_p95_ms": 4.947291221469641, + "round_gap_p50_ms": 4.542583134025335, + "round_gap_p95_ms": 4.947291221469641, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08260325016453862, + "decode_tps": 223.95881391683585, + "prefill_tps": 641.6212424381429, + "itl_mean_ms": 4.46510669757046, + "itl_p50_ms": 4.518208093941212, + "itl_p95_ms": 4.9329581670463085, + "round_gap_p50_ms": 4.526750184595585, + "round_gap_p95_ms": 4.9329581670463085, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.062266250140964985, + "decode_tps": 220.0050759252142, + "prefill_tps": 851.1834240862897, + "itl_mean_ms": 4.545349673386298, + "itl_p50_ms": 4.526833072304726, + "itl_p95_ms": 4.948209039866924, + "round_gap_p50_ms": 4.537791945040226, + "round_gap_p95_ms": 4.948209039866924, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.58547336496778, + "itl_p50_ms": 4.526750184595585, + "itl_p95_ms": 4.937666933983564, + "round_gap_p50_ms": 4.538124892860651, + "round_gap_p95_ms": 4.941083025187254, + "round_gap_mean_ms": 4.677182832267135, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2562.324461827283 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 850.0919460776175, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2213911670260131, + "end_to_end_tps": 838.3882474714105, + "timing_valid": true, + "steady_tps": 895.0945740234354, + "per_stream_tps": [ + 224.0, + 216.3, + 220.0, + 212.6 + ], + "ttft_s": [ + 0.08, + 0.04, + 0.06, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.08259270805865526, + "decode_tps": 224.02734240102964, + "prefill_tps": 641.7031387608787, + "itl_mean_ms": 4.463740850926614, + "itl_p50_ms": 4.519542213529348, + "itl_p95_ms": 4.937916994094849, + "round_gap_p50_ms": 4.536167252808809, + "round_gap_p95_ms": 4.937916994094849, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04162849998101592, + "decode_tps": 216.25294227131053, + "prefill_tps": 1273.1662208383652, + "itl_mean_ms": 4.624214540144392, + "itl_p50_ms": 4.527333192527294, + "itl_p95_ms": 4.939875099807978, + "round_gap_p50_ms": 4.5341672375798225, + "round_gap_p95_ms": 4.939875099807978, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06203712522983551, + "decode_tps": 220.03429518237607, + "prefill_tps": 854.3271436844516, + "itl_mean_ms": 4.544746077747321, + "itl_p50_ms": 4.520583897829056, + "itl_p95_ms": 4.938583355396986, + "round_gap_p50_ms": 4.54079220071435, + "round_gap_p95_ms": 4.938583355396986, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021216375287622213, + "decode_tps": 212.55703612692056, + "prefill_tps": 2498.0704423587654, + "itl_mean_ms": 4.704619608089035, + "itl_p50_ms": 4.526708275079727, + "itl_p95_ms": 4.936999641358852, + "round_gap_p50_ms": 4.533666651695967, + "round_gap_p95_ms": 4.936999641358852, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.584330269226841, + "itl_p50_ms": 4.526415839791298, + "itl_p95_ms": 4.9387919716537, + "round_gap_p50_ms": 4.54079220071435, + "round_gap_p95_ms": 4.938916768878698, + "round_gap_mean_ms": 4.676016874611378, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2565.580366463151 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 335.5, + "per_stream_tps": 335.5, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 335.479, + "prefill_tps": 2550.865, + "itl_mean_ms": 2.981, + "itl_p50_ms": 3.194, + "itl_p95_ms": 3.651, + "round_gap_p50_ms": 3.248, + "round_gap_p95_ms": 3.656, + "tokens_per_chunk": 1.103 + }, + "pooled": { + "aggregate_prefill_tps": 2550.865, + "itl_mean_ms": 2.981, + "itl_p50_ms": 3.194, + "itl_p95_ms": 3.651, + "round_gap_p50_ms": 3.248, + "round_gap_p95_ms": 3.656, + "tokens_per_chunk": 1.103 + }, + "timing_valid": true, + "end_to_end_tps": 327.65392727002364, + "makespan_s": 0.7813121671788394, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 335.479469886175, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7813121671788394, + "end_to_end_tps": 327.65392727002364, + "timing_valid": true, + "per_stream_tps": [ + 335.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021154332906007767, + "decode_tps": 335.479469886175, + "prefill_tps": 2552.668535563424, + "itl_mean_ms": 2.980808334826839, + "itl_p50_ms": 3.1951656565070152, + "itl_p95_ms": 3.6513330414891243, + "round_gap_p50_ms": 3.243750426918268, + "round_gap_p95_ms": 3.6562089808285236, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 2.980808334826839, + "itl_p50_ms": 3.1951656565070152, + "itl_p95_ms": 3.6513330414891243, + "round_gap_p50_ms": 3.243750426918268, + "round_gap_p95_ms": 3.6562089808285236, + "round_gap_mean_ms": 3.2905027072763806, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2552.668535563424 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 335.39388638845065, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7815410830080509, + "end_to_end_tps": 327.55795640926385, + "timing_valid": true, + "per_stream_tps": [ + 335.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021189458202570677, + "decode_tps": 335.39388638845065, + "prefill_tps": 2548.4370333474967, + "itl_mean_ms": 2.9815689569302632, + "itl_p50_ms": 3.1877500005066395, + "itl_p95_ms": 3.6522909067571163, + "round_gap_p50_ms": 3.248250111937523, + "round_gap_p95_ms": 3.6577079445123672, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 2.9815689569302632, + "itl_p50_ms": 3.1877500005066395, + "itl_p95_ms": 3.6522909067571163, + "round_gap_p50_ms": 3.248250111937523, + "round_gap_p95_ms": 3.6577079445123672, + "round_gap_mean_ms": 3.291342355052888, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2548.4370333474967 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 335.51748641338133, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7812399999238551, + "end_to_end_tps": 327.6841943896261, + "timing_valid": true, + "per_stream_tps": [ + 335.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02116929180920124, + "decode_tps": 335.51748641338133, + "prefill_tps": 2550.86473778631, + "itl_mean_ms": 2.9804705879559705, + "itl_p50_ms": 3.193708136677742, + "itl_p95_ms": 3.6488748155534267, + "round_gap_p50_ms": 3.257583361119032, + "round_gap_p95_ms": 3.6515407264232635, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 2.9804705879559705, + "itl_p50_ms": 3.193708136677742, + "itl_p95_ms": 3.6488748155534267, + "round_gap_p50_ms": 3.257583361119032, + "round_gap_p95_ms": 3.6515407264232635, + "round_gap_mean_ms": 3.2901298698215258, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2550.86473778631 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 869.5, + "per_stream_tps": 223.4, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.053, + "decode_tps": 223.347, + "prefill_tps": 1069.542, + "itl_mean_ms": 4.478, + "itl_p50_ms": 4.561, + "itl_p95_ms": 4.964, + "round_gap_p50_ms": 4.595, + "round_gap_p95_ms": 4.968, + "tokens_per_chunk": 1.08 + }, + "pooled": { + "aggregate_prefill_tps": 2580.648, + "itl_mean_ms": 4.477, + "itl_p50_ms": 4.558, + "itl_p95_ms": 4.963, + "round_gap_p50_ms": 4.602, + "round_gap_p95_ms": 4.967, + "tokens_per_chunk": 1.08 + }, + "timing_valid": true, + "end_to_end_tps": 857.0386564674995, + "makespan_s": 1.194811916910112, + "steady_tps": 917.4, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 869.493994465296, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1946777501143515, + "end_to_end_tps": 857.1349051256586, + "timing_valid": true, + "steady_tps": 917.545960333228, + "per_stream_tps": [ + 225.3, + 221.4, + 229.5, + 217.5 + ], + "ttft_s": [ + 0.06, + 0.04, + 0.08, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.06291695823892951, + "decode_tps": 225.34910872602268, + "prefill_tps": 858.2741682287465, + "itl_mean_ms": 4.4375591527889755, + "itl_p50_ms": 4.563583061099052, + "itl_p95_ms": 4.967750050127506, + "round_gap_p50_ms": 4.6087088994681835, + "round_gap_p95_ms": 4.971625283360481, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.042114249896258116, + "decode_tps": 221.36776195489125, + "prefill_tps": 1282.226327977361, + "itl_mean_ms": 4.517369607792181, + "itl_p50_ms": 4.558749962598085, + "itl_p95_ms": 4.970083013176918, + "round_gap_p50_ms": 4.60495799779892, + "round_gap_p95_ms": 4.9788327887654305, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.0835679997690022, + "decode_tps": 229.54852926952455, + "prefill_tps": 646.1803579033391, + "itl_mean_ms": 4.356377290598318, + "itl_p50_ms": 4.563374910503626, + "itl_p95_ms": 4.968042019754648, + "round_gap_p50_ms": 4.587208852171898, + "round_gap_p95_ms": 4.968167282640934, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021459416020661592, + "decode_tps": 217.45776524359061, + "prefill_tps": 2516.377889687568, + "itl_mean_ms": 4.598594117252266, + "itl_p50_ms": 4.5726667158305645, + "itl_p95_ms": 4.984375089406967, + "round_gap_p50_ms": 4.594708792865276, + "round_gap_p95_ms": 4.984667059034109, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.477475042107935, + "itl_p50_ms": 4.563374910503626, + "itl_p95_ms": 4.968042019754648, + "round_gap_p50_ms": 4.601750057190657, + "round_gap_p95_ms": 4.971625283360481, + "round_gap_mean_ms": 4.83794972770137, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2581.283822286187 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 869.6867178179517, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.194811916910112, + "end_to_end_tps": 857.0386564674995, + "timing_valid": true, + "steady_tps": 917.4144248035468, + "per_stream_tps": [ + 225.4, + 221.3, + 217.4, + 229.6 + ], + "ttft_s": [ + 0.06, + 0.04, + 0.02, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.06293025007471442, + "decode_tps": 225.40045874379172, + "prefill_tps": 858.0928875364087, + "itl_mean_ms": 4.43654820213423, + "itl_p50_ms": 4.552708938717842, + "itl_p95_ms": 4.95241628959775, + "round_gap_p50_ms": 4.594583995640278, + "round_gap_p95_ms": 4.9622091464698315, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04216083325445652, + "decode_tps": 221.34510399371862, + "prefill_tps": 1280.8096005619634, + "itl_mean_ms": 4.517832027711704, + "itl_p50_ms": 4.56429086625576, + "itl_p95_ms": 4.948665853589773, + "round_gap_p50_ms": 4.593667108565569, + "round_gap_p95_ms": 4.967416170984507, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.02133008325472474, + "decode_tps": 217.42678534177955, + "prefill_tps": 2531.6356882028895, + "itl_mean_ms": 4.599249344684329, + "itl_p50_ms": 4.5628328807652, + "itl_p95_ms": 4.95145795866847, + "round_gap_p50_ms": 4.591917153447866, + "round_gap_p95_ms": 4.954709205776453, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08371012471616268, + "decode_tps": 229.60388711455101, + "prefill_tps": 645.0832582450295, + "itl_mean_ms": 4.355326961433771, + "itl_p50_ms": 4.554542247205973, + "itl_p95_ms": 4.952625371515751, + "round_gap_p50_ms": 4.595291800796986, + "round_gap_p95_ms": 4.957542289048433, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.4772391339910085, + "itl_p50_ms": 4.55762492492795, + "itl_p95_ms": 4.950791131705046, + "round_gap_p50_ms": 4.594583995640278, + "round_gap_p95_ms": 4.954709205776453, + "round_gap_mean_ms": 4.83769482698181, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2580.333032980118 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 869.4226610079285, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1951397079974413, + "end_to_end_tps": 856.8035963894125, + "timing_valid": true, + "steady_tps": 917.1131813671747, + "per_stream_tps": [ + 225.3, + 221.3, + 229.5, + 217.4 + ], + "ttft_s": [ + 0.06, + 0.04, + 0.08, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.06293295929208398, + "decode_tps": 225.32630908699505, + "prefill_tps": 858.0559472720106, + "itl_mean_ms": 4.438008167141793, + "itl_p50_ms": 4.553291946649551, + "itl_p95_ms": 4.963790997862816, + "round_gap_p50_ms": 4.605249967426062, + "round_gap_p95_ms": 4.977416712790728, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.0421464997343719, + "decode_tps": 221.2729260710624, + "prefill_tps": 1281.2451885763876, + "itl_mean_ms": 4.519305717857445, + "itl_p50_ms": 4.563957918435335, + "itl_p95_ms": 4.965625237673521, + "round_gap_p50_ms": 4.592333920300007, + "round_gap_p95_ms": 4.979624878615141, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08358362503349781, + "decode_tps": 229.52685178932524, + "prefill_tps": 646.0595598522847, + "itl_mean_ms": 4.356788725172188, + "itl_p50_ms": 4.5482078567147255, + "itl_p95_ms": 4.964000079780817, + "round_gap_p50_ms": 4.59058303385973, + "round_gap_p95_ms": 4.9673751927912235, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.0214887079782784, + "decode_tps": 217.36079098871784, + "prefill_tps": 2512.947733041244, + "itl_mean_ms": 4.600645753317604, + "itl_p50_ms": 4.559332970529795, + "itl_p95_ms": 4.956874996423721, + "round_gap_p50_ms": 4.604083951562643, + "round_gap_p95_ms": 4.967415705323219, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.478687090872257, + "itl_p50_ms": 4.55420883372426, + "itl_p95_ms": 4.9628340639173985, + "round_gap_p50_ms": 4.602166824042797, + "round_gap_p95_ms": 4.9673751927912235, + "round_gap_mean_ms": 4.83925935666282, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2580.6477504551035 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + } + ] + } + ] + } + }, + { + "name": "text-4-2-new.concurrent.json", + "data": { + "label": "text-4-2-new", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 320.2, + "per_stream_tps": 320.2, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 320.236, + "prefill_tps": 2536.846, + "itl_mean_ms": 3.123, + "itl_p50_ms": 3.178, + "itl_p95_ms": 3.648, + "round_gap_p50_ms": 3.192, + "round_gap_p95_ms": 3.648, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2536.846, + "itl_mean_ms": 3.123, + "itl_p50_ms": 3.178, + "itl_p95_ms": 3.648, + "round_gap_p50_ms": 3.192, + "round_gap_p95_ms": 3.648, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 313.2552235119364, + "makespan_s": 0.8172249998897314, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 320.83793626042257, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.815777416806668, + "end_to_end_tps": 313.8110895519797, + "timing_valid": true, + "per_stream_tps": [ + 320.8 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.020927749574184418, + "decode_tps": 320.83793626042257, + "prefill_tps": 2532.5226590716925, + "itl_mean_ms": 3.116838400270425, + "itl_p50_ms": 3.163333050906658, + "itl_p95_ms": 3.6288751289248466, + "round_gap_p50_ms": 3.1851669773459435, + "round_gap_p95_ms": 3.6288751289248466, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.116838400270425, + "itl_p50_ms": 3.163333050906658, + "itl_p95_ms": 3.6288751289248466, + "round_gap_p50_ms": 3.1851669773459435, + "round_gap_p95_ms": 3.6288751289248466, + "round_gap_mean_ms": 3.191942940035977, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2532.5226590716925 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 320.13866821131575, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8174729580059648, + "end_to_end_tps": 313.1602060873701, + "timing_valid": true, + "per_stream_tps": [ + 320.1 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.020883500110358, + "decode_tps": 320.13866821131575, + "prefill_tps": 2537.888750445264, + "itl_mean_ms": 3.123646404813318, + "itl_p50_ms": 3.1842091120779514, + "itl_p95_ms": 3.657624591141939, + "round_gap_p50_ms": 3.1964173540472984, + "round_gap_p95_ms": 3.657624591141939, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.123646404813318, + "itl_p50_ms": 3.1842091120779514, + "itl_p95_ms": 3.657624591141939, + "round_gap_p50_ms": 3.1964173540472984, + "round_gap_p95_ms": 3.657624591141939, + "round_gap_mean_ms": 3.1989149928811087, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2537.888750445264 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 320.23646406057856, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8172249998897314, + "end_to_end_tps": 313.2552235119364, + "timing_valid": true, + "per_stream_tps": [ + 320.2 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.020892083179205656, + "decode_tps": 320.23646406057856, + "prefill_tps": 2536.8461127300147, + "itl_mean_ms": 3.1226924857964704, + "itl_p50_ms": 3.177833743393421, + "itl_p95_ms": 3.6481251008808613, + "round_gap_p50_ms": 3.1919581815600395, + "round_gap_p95_ms": 3.6481251008808613, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.1226924857964704, + "itl_p50_ms": 3.177833743393421, + "itl_p95_ms": 3.6481251008808613, + "round_gap_p50_ms": 3.1919581815600395, + "round_gap_p95_ms": 3.6481251008808613, + "round_gap_mean_ms": 3.197938087863855, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2536.8461127300147 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 849.2, + "per_stream_tps": 217.9, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.052, + "decode_tps": 217.879, + "prefill_tps": 1063.305, + "itl_mean_ms": 4.59, + "itl_p50_ms": 4.531, + "itl_p95_ms": 4.935, + "round_gap_p50_ms": 4.54, + "round_gap_p95_ms": 4.935, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2565.787, + "itl_mean_ms": 4.589, + "itl_p50_ms": 4.531, + "itl_p95_ms": 4.934, + "round_gap_p50_ms": 4.543, + "round_gap_p95_ms": 4.935, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 837.4392316980054, + "makespan_s": 1.2227752907201648, + "steady_tps": 894.1, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 848.3939430079084, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2234192504547536, + "end_to_end_tps": 836.9984366515174, + "timing_valid": true, + "steady_tps": 893.4669019856921, + "per_stream_tps": [ + 223.5, + 212.2, + 215.9, + 219.7 + ], + "ttft_s": [ + 0.08, + 0.02, + 0.04, + 0.06 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.08254654193297029, + "decode_tps": 223.52401008898724, + "prefill_tps": 642.0620265720789, + "itl_mean_ms": 4.473792321468685, + "itl_p50_ms": 4.530542064458132, + "itl_p95_ms": 4.934541881084442, + "round_gap_p50_ms": 4.541208036243916, + "round_gap_p95_ms": 4.934541881084442, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021045958157628775, + "decode_tps": 212.18767158094428, + "prefill_tps": 2518.2982691043917, + "itl_mean_ms": 4.712809149321973, + "itl_p50_ms": 4.532249644398689, + "itl_p95_ms": 4.932708106935024, + "round_gap_p50_ms": 4.5382496900856495, + "round_gap_p95_ms": 4.932708106935024, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04150220798328519, + "decode_tps": 215.8569502275108, + "prefill_tps": 1277.0404895408333, + "itl_mean_ms": 4.632697714602246, + "itl_p50_ms": 4.524957854300737, + "itl_p95_ms": 4.934333730489016, + "round_gap_p50_ms": 4.537124652415514, + "round_gap_p95_ms": 4.934333730489016, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06211191602051258, + "decode_tps": 219.67661364103824, + "prefill_tps": 853.2984231640294, + "itl_mean_ms": 4.5521459176990335, + "itl_p50_ms": 4.539959132671356, + "itl_p95_ms": 4.939582664519548, + "round_gap_p50_ms": 4.549582954496145, + "round_gap_p95_ms": 4.939582664519548, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.592861275772984, + "itl_p50_ms": 4.531290847808123, + "itl_p95_ms": 4.934499971568584, + "round_gap_p50_ms": 4.545582924038172, + "round_gap_p95_ms": 4.934541881084442, + "round_gap_mean_ms": 4.684718501288444, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2567.026234500813 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 849.283004781087, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2227752907201648, + "end_to_end_tps": 837.4392316980054, + "timing_valid": true, + "steady_tps": 894.0892935348813, + "per_stream_tps": [ + 223.8, + 219.8, + 216.0, + 212.3 + ], + "ttft_s": [ + 0.08, + 0.06, + 0.04, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.08270312519744039, + "decode_tps": 223.77486544514963, + "prefill_tps": 640.8463993769406, + "itl_mean_ms": 4.4687771256647855, + "itl_p50_ms": 4.537417087703943, + "itl_p95_ms": 4.9525839276611805, + "round_gap_p50_ms": 4.539750050753355, + "round_gap_p95_ms": 4.9525839276611805, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06216087518259883, + "decode_tps": 219.81762967283865, + "prefill_tps": 852.6263480736303, + "itl_mean_ms": 4.549225653503456, + "itl_p50_ms": 4.53091599047184, + "itl_p95_ms": 4.944457672536373, + "round_gap_p50_ms": 4.542041104286909, + "round_gap_p95_ms": 4.944457672536373, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04162774980068207, + "decode_tps": 216.00835057277078, + "prefill_tps": 1273.189164770362, + "itl_mean_ms": 4.629450654793604, + "itl_p50_ms": 4.537208005785942, + "itl_p95_ms": 4.938959144055843, + "round_gap_p50_ms": 4.5379577204585075, + "round_gap_p95_ms": 4.938959144055843, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.02128112455829978, + "decode_tps": 212.3253844493911, + "prefill_tps": 2490.4698929234755, + "itl_mean_ms": 4.7097524518475815, + "itl_p50_ms": 4.533708095550537, + "itl_p95_ms": 4.951042123138905, + "round_gap_p50_ms": 4.545334260910749, + "round_gap_p95_ms": 4.951042123138905, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.589301471452356, + "itl_p50_ms": 4.5350827276706696, + "itl_p95_ms": 4.944457672536373, + "round_gap_p50_ms": 4.542707931250334, + "round_gap_p95_ms": 4.946416709572077, + "round_gap_mean_ms": 4.681087500881404, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2561.9309667034227 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 849.1967419477742, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2223647092469037, + "end_to_end_tps": 837.7205201145608, + "timing_valid": true, + "steady_tps": 894.3790434225432, + "per_stream_tps": [ + 223.7, + 219.9, + 216.1, + 212.4 + ], + "ttft_s": [ + 0.08, + 0.06, + 0.04, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.0825841249898076, + "decode_tps": 223.73994919237222, + "prefill_tps": 641.76983175085, + "itl_mean_ms": 4.469474510965394, + "itl_p50_ms": 4.525083117187023, + "itl_p95_ms": 4.9303751438856125, + "round_gap_p50_ms": 4.540000110864639, + "round_gap_p95_ms": 4.9303751438856125, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.062102957628667355, + "decode_tps": 219.8891534036551, + "prefill_tps": 853.4215120140215, + "itl_mean_ms": 4.547745918891594, + "itl_p50_ms": 4.5309169217944145, + "itl_p95_ms": 4.9271658062934875, + "round_gap_p50_ms": 4.545917268842459, + "round_gap_p95_ms": 4.9271658062934875, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04156904201954603, + "decode_tps": 216.081704414024, + "prefill_tps": 1274.9872844093702, + "itl_mean_ms": 4.627879082645271, + "itl_p50_ms": 4.5302920043468475, + "itl_p95_ms": 4.9274577759206295, + "round_gap_p50_ms": 4.533458035439253, + "round_gap_p95_ms": 4.9274577759206295, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021206625271588564, + "decode_tps": 212.39949368997526, + "prefill_tps": 2499.2189620574095, + "itl_mean_ms": 4.708109151425899, + "itl_p50_ms": 4.523625131696463, + "itl_p95_ms": 4.936249926686287, + "round_gap_p50_ms": 4.530957899987698, + "round_gap_p95_ms": 4.936249926686287, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.588302165982039, + "itl_p50_ms": 4.525959026068449, + "itl_p95_ms": 4.9303751438856125, + "round_gap_p50_ms": 4.540000110864639, + "round_gap_p95_ms": 4.931750241667032, + "round_gap_mean_ms": 4.68006820930168, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2565.787362603836 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 335.7, + "per_stream_tps": 335.7, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 335.678, + "prefill_tps": 2548.622, + "itl_mean_ms": 2.979, + "itl_p50_ms": 3.196, + "itl_p95_ms": 3.66, + "round_gap_p50_ms": 3.244, + "round_gap_p95_ms": 3.661, + "tokens_per_chunk": 1.103 + }, + "pooled": { + "aggregate_prefill_tps": 2548.622, + "itl_mean_ms": 2.979, + "itl_p50_ms": 3.196, + "itl_p95_ms": 3.66, + "round_gap_p50_ms": 3.244, + "round_gap_p95_ms": 3.661, + "tokens_per_chunk": 1.103 + }, + "timing_valid": true, + "end_to_end_tps": 327.7783164038572, + "makespan_s": 0.7810156657360494, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 334.4428419268118, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7837137919850647, + "end_to_end_tps": 326.64985944879044, + "timing_valid": true, + "per_stream_tps": [ + 334.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021187916863709688, + "decode_tps": 334.4428419268118, + "prefill_tps": 2548.6224222679625, + "itl_mean_ms": 2.990047549646275, + "itl_p50_ms": 3.2009161077439785, + "itl_p95_ms": 3.659666981548071, + "round_gap_p50_ms": 3.2660001888871193, + "round_gap_p95_ms": 3.6607920192182064, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 2.990047549646275, + "itl_p50_ms": 3.2009161077439785, + "itl_p95_ms": 3.659666981548071, + "round_gap_p50_ms": 3.2660001888871193, + "round_gap_p95_ms": 3.6607920192182064, + "round_gap_mean_ms": 3.300701840518615, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2548.6224222679625 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 336.62170188970384, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7787955002859235, + "end_to_end_tps": 328.7127364064293, + "timing_valid": true, + "per_stream_tps": [ + 336.6 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021173041313886642, + "decode_tps": 336.62170188970384, + "prefill_tps": 2550.4130086679293, + "itl_mean_ms": 2.970693791832994, + "itl_p50_ms": 3.176833037286997, + "itl_p95_ms": 3.648709040135145, + "round_gap_p50_ms": 3.2180831767618656, + "round_gap_p95_ms": 3.649457823485136, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 2.970693791832994, + "itl_p50_ms": 3.176833037286997, + "itl_p95_ms": 3.648709040135145, + "round_gap_p50_ms": 3.2180831767618656, + "round_gap_p95_ms": 3.649457823485136, + "round_gap_mean_ms": 3.2793373026727854, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2550.4130086679293 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 335.67836488646947, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7810156657360494, + "end_to_end_tps": 327.7783164038572, + "timing_valid": true, + "per_stream_tps": [ + 335.7 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021305915899574757, + "decode_tps": 335.67836488646947, + "prefill_tps": 2534.507329069002, + "itl_mean_ms": 2.979042156435111, + "itl_p50_ms": 3.1957500614225864, + "itl_p95_ms": 3.6598751321434975, + "round_gap_p50_ms": 3.244000021368265, + "round_gap_p95_ms": 3.6609582602977753, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 2.979042156435111, + "itl_p50_ms": 3.1957500614225864, + "itl_p95_ms": 3.6598751321434975, + "round_gap_p50_ms": 3.244000021368265, + "round_gap_p95_ms": 3.6609582602977753, + "round_gap_mean_ms": 3.288553029830967, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2534.507329069002 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 868.0, + "per_stream_tps": 222.8, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.053, + "decode_tps": 222.821, + "prefill_tps": 1069.147, + "itl_mean_ms": 4.488, + "itl_p50_ms": 4.565, + "itl_p95_ms": 4.975, + "round_gap_p50_ms": 4.605, + "round_gap_p95_ms": 4.982, + "tokens_per_chunk": 1.08 + }, + "pooled": { + "aggregate_prefill_tps": 2580.649, + "itl_mean_ms": 4.486, + "itl_p50_ms": 4.563, + "itl_p95_ms": 4.969, + "round_gap_p50_ms": 4.607, + "round_gap_p95_ms": 4.979, + "tokens_per_chunk": 1.08 + }, + "timing_valid": true, + "end_to_end_tps": 855.4271696296133, + "makespan_s": 1.1970627498812973, + "steady_tps": 915.5, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 866.7257301721808, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1987081253901124, + "end_to_end_tps": 854.2529897899418, + "timing_valid": true, + "steady_tps": 914.2430704785231, + "per_stream_tps": [ + 228.8, + 220.6, + 224.6, + 216.7 + ], + "ttft_s": [ + 0.08, + 0.04, + 0.06, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.08364270813763142, + "decode_tps": 228.78755274601437, + "prefill_tps": 645.6031996374952, + "itl_mean_ms": 4.37086715600362, + "itl_p50_ms": 4.565124865621328, + "itl_p95_ms": 4.982125014066696, + "round_gap_p50_ms": 4.611792042851448, + "round_gap_p95_ms": 4.986708052456379, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04210070800036192, + "decode_tps": 220.5934511006221, + "prefill_tps": 1282.6387622634704, + "itl_mean_ms": 4.533226145248787, + "itl_p50_ms": 4.569417331367731, + "itl_p95_ms": 4.975958727300167, + "round_gap_p50_ms": 4.614749923348427, + "round_gap_p95_ms": 5.004000384360552, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06279224995523691, + "decode_tps": 224.6114387003433, + "prefill_tps": 859.978740027556, + "itl_mean_ms": 4.452133007055404, + "itl_p50_ms": 4.56741638481617, + "itl_p95_ms": 4.982207901775837, + "round_gap_p50_ms": 4.610749892890453, + "round_gap_p95_ms": 4.98370872810483, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021410208195447922, + "decode_tps": 216.70682116758414, + "prefill_tps": 2522.1613684018766, + "itl_mean_ms": 4.614529411728475, + "itl_p50_ms": 4.56962501630187, + "itl_p95_ms": 4.973333328962326, + "round_gap_p50_ms": 4.6112919226288795, + "round_gap_p95_ms": 4.993041977286339, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.492688930009072, + "itl_p50_ms": 4.569417331367731, + "itl_p95_ms": 4.9772500060498714, + "round_gap_p50_ms": 4.613665863871574, + "round_gap_p95_ms": 4.98370872810483, + "round_gap_mean_ms": 4.854388462509802, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2581.201545547511 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 868.4718223952849, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1964600002393126, + "end_to_end_tps": 855.8581146007241, + "timing_valid": true, + "steady_tps": 916.0637651936847, + "per_stream_tps": [ + 225.1, + 217.1, + 229.3, + 221.0 + ], + "ttft_s": [ + 0.06, + 0.02, + 0.08, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.0629321662709117, + "decode_tps": 225.07329773436786, + "prefill_tps": 858.0667598115037, + "itl_mean_ms": 4.442997059474389, + "itl_p50_ms": 4.563124850392342, + "itl_p95_ms": 4.97354194521904, + "round_gap_p50_ms": 4.6021658927202225, + "round_gap_p95_ms": 4.974833223968744, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021399250254034996, + "decode_tps": 217.12357099040332, + "prefill_tps": 2523.452894795596, + "itl_mean_ms": 4.605672223603024, + "itl_p50_ms": 4.560499917715788, + "itl_p95_ms": 4.966416861861944, + "round_gap_p50_ms": 4.605291876941919, + "round_gap_p95_ms": 4.979999735951424, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08356820791959763, + "decode_tps": 229.26527812912332, + "prefill_tps": 646.178748405785, + "itl_mean_ms": 4.361759478628051, + "itl_p50_ms": 4.567124880850315, + "itl_p95_ms": 4.960500169545412, + "round_gap_p50_ms": 4.60429210215807, + "round_gap_p95_ms": 4.9625420942902565, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04224312538281083, + "decode_tps": 221.03035993216903, + "prefill_tps": 1278.3145070504931, + "itl_mean_ms": 4.524265355704462, + "itl_p50_ms": 4.563749767839909, + "itl_p95_ms": 4.964832682162523, + "round_gap_p50_ms": 4.606666974723339, + "round_gap_p95_ms": 4.9727498553693295, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.483673529352481, + "itl_p50_ms": 4.563374910503626, + "itl_p95_ms": 4.965250380337238, + "round_gap_p50_ms": 4.606666974723339, + "round_gap_p95_ms": 4.9727498553693295, + "round_gap_mean_ms": 4.844647245698656, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2580.636178494794 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 867.9890732996824, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1970627498812973, + "end_to_end_tps": 855.4271696296133, + "timing_valid": true, + "steady_tps": 915.535604849051, + "per_stream_tps": [ + 220.9, + 217.0, + 224.9, + 229.1 + ], + "ttft_s": [ + 0.04, + 0.02, + 0.06, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.042198124807327986, + "decode_tps": 220.90969463184348, + "prefill_tps": 1279.67771664163, + "itl_mean_ms": 4.526736600068855, + "itl_p50_ms": 4.561417270451784, + "itl_p95_ms": 4.9782078713178635, + "round_gap_p50_ms": 4.598415922373533, + "round_gap_p95_ms": 4.981291946023703, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021402083802968264, + "decode_tps": 217.00085391707225, + "prefill_tps": 2523.1187998857718, + "itl_mean_ms": 4.6082767968376475, + "itl_p50_ms": 4.567082971334457, + "itl_p95_ms": 4.978584125638008, + "round_gap_p50_ms": 4.605207592248917, + "round_gap_p95_ms": 4.981915932148695, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06283600023016334, + "decode_tps": 224.9425321506639, + "prefill_tps": 859.3799701158928, + "itl_mean_ms": 4.445579901848938, + "itl_p50_ms": 4.565707873553038, + "itl_p95_ms": 4.979041870683432, + "round_gap_p50_ms": 4.601624794304371, + "round_gap_p95_ms": 4.99429227784276, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08369987504556775, + "decode_tps": 229.13004373968255, + "prefill_tps": 645.1622534753058, + "itl_mean_ms": 4.364333824053699, + "itl_p50_ms": 4.555374849587679, + "itl_p95_ms": 4.958916921168566, + "round_gap_p50_ms": 4.603041335940361, + "round_gap_p95_ms": 4.968832712620497, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.486231780702285, + "itl_p50_ms": 4.563209135085344, + "itl_p95_ms": 4.968832712620497, + "round_gap_p50_ms": 4.604875110089779, + "round_gap_p95_ms": 4.979041870683432, + "round_gap_mean_ms": 4.847411457962214, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2580.649013901223 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + } + ] + } + ] + } + }, + { + "name": "text-4-3-old.concurrent.json", + "data": { + "label": "text-4-3-old", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 316.7, + "per_stream_tps": 316.7, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 316.733, + "prefill_tps": 2531.504, + "itl_mean_ms": 3.157, + "itl_p50_ms": 3.218, + "itl_p95_ms": 3.676, + "round_gap_p50_ms": 3.238, + "round_gap_p95_ms": 3.676, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2531.504, + "itl_mean_ms": 3.157, + "itl_p50_ms": 3.218, + "itl_p95_ms": 3.676, + "round_gap_p50_ms": 3.238, + "round_gap_p95_ms": 3.676, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 309.89407966646144, + "makespan_s": 0.8260887083597481, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 316.73302072768723, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8260887083597481, + "end_to_end_tps": 309.89407966646144, + "timing_valid": true, + "per_stream_tps": [ + 316.7 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.0209345412440598, + "decode_tps": 316.73302072768723, + "prefill_tps": 2531.701047666321, + "itl_mean_ms": 3.157233172918067, + "itl_p50_ms": 3.2180417329072952, + "itl_p95_ms": 3.675582818686962, + "round_gap_p50_ms": 3.2377499155700207, + "round_gap_p95_ms": 3.675582818686962, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.157233172918067, + "itl_p50_ms": 3.2180417329072952, + "itl_p95_ms": 3.675582818686962, + "round_gap_p50_ms": 3.2377499155700207, + "round_gap_p95_ms": 3.675582818686962, + "round_gap_mean_ms": 3.2333110806992256, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2531.701047666321 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 316.3524603194822, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8270497089251876, + "end_to_end_tps": 309.5339944350999, + "timing_valid": true, + "per_stream_tps": [ + 316.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.020936166867613792, + "decode_tps": 316.3524603194822, + "prefill_tps": 2531.5044695209144, + "itl_mean_ms": 3.16103120864022, + "itl_p50_ms": 3.237874712795019, + "itl_p95_ms": 3.684207797050476, + "round_gap_p50_ms": 3.247541841119528, + "round_gap_p95_ms": 3.684207797050476, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.16103120864022, + "itl_p50_ms": 3.237874712795019, + "itl_p95_ms": 3.684207797050476, + "round_gap_p50_ms": 3.247541841119528, + "round_gap_p95_ms": 3.684207797050476, + "round_gap_mean_ms": 3.2372006353544425, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2531.5044695209144 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 317.13140737741855, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8250718326307833, + "end_to_end_tps": 310.2760146152742, + "timing_valid": true, + "per_stream_tps": [ + 317.1 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02093841601163149, + "decode_tps": 317.13140737741855, + "prefill_tps": 2531.2325426411435, + "itl_mean_ms": 3.1532669951226198, + "itl_p50_ms": 3.210250288248062, + "itl_p95_ms": 3.6662910133600235, + "round_gap_p50_ms": 3.227416891604662, + "round_gap_p95_ms": 3.6662910133600235, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.1532669951226198, + "itl_p50_ms": 3.210250288248062, + "itl_p95_ms": 3.6662910133600235, + "round_gap_p50_ms": 3.227416891604662, + "round_gap_p95_ms": 3.6662910133600235, + "round_gap_mean_ms": 3.22924933235449, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2531.2325426411435 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 838.8, + "per_stream_tps": 215.1, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.052, + "decode_tps": 215.055, + "prefill_tps": 1062.581, + "itl_mean_ms": 4.65, + "itl_p50_ms": 4.594, + "itl_p95_ms": 5.006, + "round_gap_p50_ms": 4.602, + "round_gap_p95_ms": 5.006, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2564.857, + "itl_mean_ms": 4.648, + "itl_p50_ms": 4.592, + "itl_p95_ms": 5.006, + "round_gap_p50_ms": 4.601, + "round_gap_p95_ms": 5.006, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 827.310288334921, + "makespan_s": 1.2377459998242557, + "steady_tps": 882.5, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 837.867963043507, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2389886667951941, + "end_to_end_tps": 826.480521931415, + "timing_valid": true, + "steady_tps": 881.4877435023707, + "per_stream_tps": [ + 216.8, + 213.1, + 220.6, + 209.5 + ], + "ttft_s": [ + 0.06, + 0.04, + 0.08, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.06204608315601945, + "decode_tps": 216.7772638020592, + "prefill_tps": 854.2037998873772, + "itl_mean_ms": 4.613029902033946, + "itl_p50_ms": 4.600250162184238, + "itl_p95_ms": 5.007999949157238, + "round_gap_p50_ms": 4.606916569173336, + "round_gap_p95_ms": 5.007999949157238, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.041503208223730326, + "decode_tps": 213.06053400770162, + "prefill_tps": 1277.0097124611236, + "itl_mean_ms": 4.69350180059087, + "itl_p50_ms": 4.6050408855080605, + "itl_p95_ms": 5.004041828215122, + "round_gap_p50_ms": 4.610709380358458, + "round_gap_p95_ms": 5.004041828215122, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08259691670536995, + "decode_tps": 220.61856399799777, + "prefill_tps": 641.6704413925692, + "itl_mean_ms": 4.532710130454278, + "itl_p50_ms": 4.607916809618473, + "itl_p95_ms": 5.003707949072123, + "round_gap_p50_ms": 4.608582705259323, + "round_gap_p95_ms": 5.003707949072123, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021121167112141848, + "decode_tps": 209.47264756862364, + "prefill_tps": 2509.3310288488788, + "itl_mean_ms": 4.773892971741803, + "itl_p50_ms": 4.605208057910204, + "itl_p95_ms": 5.010874941945076, + "round_gap_p50_ms": 4.613083321601152, + "round_gap_p95_ms": 5.010874941945076, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.653283701205225, + "itl_p50_ms": 4.604707937687635, + "itl_p95_ms": 5.0056250765919685, + "round_gap_p50_ms": 4.610709380358458, + "round_gap_p95_ms": 5.006374791264534, + "round_gap_mean_ms": 4.746349375229329, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2564.857414492379 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 838.751842242589, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2377459998242557, + "end_to_end_tps": 827.310288334921, + "timing_valid": true, + "steady_tps": 882.475635264268, + "per_stream_tps": [ + 209.7, + 217.0, + 220.9, + 213.3 + ], + "ttft_s": [ + 0.02, + 0.06, + 0.08, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02114670816808939, + "decode_tps": 209.70695812784243, + "prefill_tps": 2506.3002514962386, + "itl_mean_ms": 4.768558987872858, + "itl_p50_ms": 4.598875064402819, + "itl_p95_ms": 5.0103748217225075, + "round_gap_p50_ms": 4.602290689945221, + "round_gap_p95_ms": 5.0103748217225075, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06210833275690675, + "decode_tps": 217.00979502441194, + "prefill_tps": 853.3476531634338, + "itl_mean_ms": 4.6080869293826465, + "itl_p50_ms": 4.594583995640278, + "itl_p95_ms": 5.008292384445667, + "round_gap_p50_ms": 4.602083005011082, + "round_gap_p95_ms": 5.008292384445667, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08253820892423391, + "decode_tps": 220.85102129216546, + "prefill_tps": 642.1268487743832, + "itl_mean_ms": 4.527939215083332, + "itl_p50_ms": 4.592834040522575, + "itl_p95_ms": 5.013334099203348, + "round_gap_p50_ms": 4.598459228873253, + "round_gap_p95_ms": 5.013334099203348, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04170691594481468, + "decode_tps": 213.29986706732055, + "prefill_tps": 1270.7724558231057, + "itl_mean_ms": 4.6882354581326835, + "itl_p50_ms": 4.5871250331401825, + "itl_p95_ms": 5.015749949961901, + "round_gap_p50_ms": 4.588666837662458, + "round_gap_p95_ms": 5.015749949961901, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.64820514761788, + "itl_p50_ms": 4.591916687786579, + "itl_p95_ms": 5.0103748217225075, + "round_gap_p50_ms": 4.601249936968088, + "round_gap_p95_ms": 5.010416265577078, + "round_gap_mean_ms": 4.741169250570238, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2564.7345973873175 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 838.9347995625418, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.237489040941, + "end_to_end_tps": 827.482075494858, + "timing_valid": true, + "steady_tps": 882.5925989550436, + "per_stream_tps": [ + 209.7, + 213.3, + 217.0, + 220.9 + ], + "ttft_s": [ + 0.02, + 0.04, + 0.06, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021169000305235386, + "decode_tps": 209.73909798383445, + "prefill_tps": 2503.6609776462788, + "itl_mean_ms": 4.7678282667024465, + "itl_p50_ms": 4.580792039632797, + "itl_p95_ms": 4.9985842779278755, + "round_gap_p50_ms": 4.59416676312685, + "round_gap_p95_ms": 4.9985842779278755, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04162820801138878, + "decode_tps": 213.33235905246596, + "prefill_tps": 1273.175150501316, + "itl_mean_ms": 4.687521407636357, + "itl_p50_ms": 4.583083093166351, + "itl_p95_ms": 4.999291151762009, + "round_gap_p50_ms": 4.6005831100046635, + "round_gap_p95_ms": 4.999291151762009, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06203254219144583, + "decode_tps": 217.04571349813773, + "prefill_tps": 854.3902623953496, + "itl_mean_ms": 4.607324346023447, + "itl_p50_ms": 4.5905825681984425, + "itl_p95_ms": 5.003917030990124, + "round_gap_p50_ms": 4.603708162903786, + "round_gap_p95_ms": 5.003917030990124, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.0826470828615129, + "decode_tps": 220.8975985819713, + "prefill_tps": 641.2809522728, + "itl_mean_ms": 4.526984477963517, + "itl_p50_ms": 4.578291904181242, + "itl_p95_ms": 4.995208699256182, + "round_gap_p50_ms": 4.5877499505877495, + "round_gap_p95_ms": 4.995208699256182, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.647414624581442, + "itl_p50_ms": 4.5823752880096436, + "itl_p95_ms": 4.998665768653154, + "round_gap_p50_ms": 4.598417319357395, + "round_gap_p95_ms": 4.999083001166582, + "round_gap_mean_ms": 4.740362917073071, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2565.1238090912 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 331.3, + "per_stream_tps": 331.3, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 331.291, + "prefill_tps": 2546.599, + "itl_mean_ms": 3.018, + "itl_p50_ms": 3.221, + "itl_p95_ms": 3.687, + "round_gap_p50_ms": 3.294, + "round_gap_p95_ms": 3.689, + "tokens_per_chunk": 1.103 + }, + "pooled": { + "aggregate_prefill_tps": 2546.599, + "itl_mean_ms": 3.018, + "itl_p50_ms": 3.221, + "itl_p95_ms": 3.687, + "round_gap_p50_ms": 3.294, + "round_gap_p95_ms": 3.689, + "tokens_per_chunk": 1.103 + }, + "timing_valid": true, + "end_to_end_tps": 323.6683936432555, + "makespan_s": 0.7909329580143094, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 331.29120072315726, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7909329580143094, + "end_to_end_tps": 323.6683936432555, + "timing_valid": true, + "per_stream_tps": [ + 331.3 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021166833117604256, + "decode_tps": 331.29120072315726, + "prefill_tps": 2551.161040481238, + "itl_mean_ms": 3.018492485816572, + "itl_p50_ms": 3.2140840776264668, + "itl_p95_ms": 3.687415737658739, + "round_gap_p50_ms": 3.2939999364316463, + "round_gap_p95_ms": 3.6888327449560165, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.018492485816572, + "itl_p50_ms": 3.2140840776264668, + "itl_p95_ms": 3.687415737658739, + "round_gap_p50_ms": 3.2939999364316463, + "round_gap_p95_ms": 3.6888327449560165, + "round_gap_mean_ms": 3.33210209473258, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2551.161040481238 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 331.2650558705951, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7910715830512345, + "end_to_end_tps": 323.61167495435103, + "timing_valid": true, + "per_stream_tps": [ + 331.3 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021204750053584576, + "decode_tps": 331.2650558705951, + "prefill_tps": 2546.5992225110676, + "itl_mean_ms": 3.0187307181311587, + "itl_p50_ms": 3.220916725695133, + "itl_p95_ms": 3.6968328058719635, + "round_gap_p50_ms": 3.2959161326289177, + "round_gap_p95_ms": 3.7000002339482307, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.0187307181311587, + "itl_p50_ms": 3.220916725695133, + "itl_p95_ms": 3.6968328058719635, + "round_gap_p50_ms": 3.2959161326289177, + "round_gap_p95_ms": 3.7000002339482307, + "round_gap_mean_ms": 3.332365078456474, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2546.5992225110676 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 332.07735368128317, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7892962917685509, + "end_to_end_tps": 324.33954481958733, + "timing_valid": true, + "per_stream_tps": [ + 332.1 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021328416652977467, + "decode_tps": 332.07735368128317, + "prefill_tps": 2531.8335101289176, + "itl_mean_ms": 3.011346570051768, + "itl_p50_ms": 3.231000155210495, + "itl_p95_ms": 3.677540924400091, + "round_gap_p50_ms": 3.2867081463336945, + "round_gap_p95_ms": 3.677875269204378, + "tokens_per_chunk": 1.103448275862069 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.011346570051768, + "itl_p50_ms": 3.231000155210495, + "itl_p95_ms": 3.677540924400091, + "round_gap_p50_ms": 3.2867081463336945, + "round_gap_p95_ms": 3.677875269204378, + "round_gap_mean_ms": 3.324213746161043, + "tokens_per_chunk": 1.103448275862069, + "aggregate_prefill_tps": 2531.8335101289176 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 858.1, + "per_stream_tps": 220.5, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.053, + "decode_tps": 220.466, + "prefill_tps": 1067.417, + "itl_mean_ms": 4.536, + "itl_p50_ms": 4.624, + "itl_p95_ms": 5.029, + "round_gap_p50_ms": 4.655, + "round_gap_p95_ms": 5.034, + "tokens_per_chunk": 1.08 + }, + "pooled": { + "aggregate_prefill_tps": 2578.658, + "itl_mean_ms": 4.538, + "itl_p50_ms": 4.625, + "itl_p95_ms": 5.025, + "round_gap_p50_ms": 4.659, + "round_gap_p95_ms": 5.034, + "tokens_per_chunk": 1.08 + }, + "timing_valid": true, + "end_to_end_tps": 846.1663335145463, + "makespan_s": 1.2101639588363469, + "steady_tps": 905.0, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 857.9184984239495, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2108651250600815, + "end_to_end_tps": 845.6763505755363, + "timing_valid": true, + "steady_tps": 904.3975377890271, + "per_stream_tps": [ + 226.3, + 218.3, + 214.5, + 222.3 + ], + "ttft_s": [ + 0.08, + 0.04, + 0.02, + 0.06 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.08372404193505645, + "decode_tps": 226.34512234851107, + "prefill_tps": 644.9760278163235, + "itl_mean_ms": 4.418032028365369, + "itl_p50_ms": 4.624083172529936, + "itl_p95_ms": 5.02933282405138, + "round_gap_p50_ms": 4.653291776776314, + "round_gap_p95_ms": 5.034625064581633, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.042074792087078094, + "decode_tps": 218.30307945176384, + "prefill_tps": 1283.4288019354074, + "itl_mean_ms": 4.580787419542378, + "itl_p50_ms": 4.627417307347059, + "itl_p95_ms": 5.034625064581633, + "round_gap_p50_ms": 4.660707898437977, + "round_gap_p95_ms": 5.036542192101479, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021358459256589413, + "decode_tps": 214.48191736259324, + "prefill_tps": 2528.2722574354316, + "itl_mean_ms": 4.662397708378586, + "itl_p50_ms": 4.626583307981491, + "itl_p95_ms": 5.032500252127647, + "round_gap_p50_ms": 4.65587479993701, + "round_gap_p95_ms": 5.042791832238436, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06299574999138713, + "decode_tps": 222.2520091555022, + "prefill_tps": 857.2006842903364, + "itl_mean_ms": 4.499396895441939, + "itl_p50_ms": 4.627916030585766, + "itl_p95_ms": 5.03487465903163, + "round_gap_p50_ms": 4.667582921683788, + "round_gap_p95_ms": 5.040375050157309, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.540153512932068, + "itl_p50_ms": 4.625583998858929, + "itl_p95_ms": 5.031582899391651, + "round_gap_p50_ms": 4.658624995499849, + "round_gap_p95_ms": 5.03487465903163, + "round_gap_mean_ms": 4.905674346600327, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2578.6580238585666 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 858.1469732662516, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2101639588363469, + "end_to_end_tps": 846.1663335145463, + "timing_valid": true, + "steady_tps": 905.007335386431, + "per_stream_tps": [ + 226.4, + 214.6, + 222.4, + 218.4 + ], + "ttft_s": [ + 0.08, + 0.02, + 0.06, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.08378829201683402, + "decode_tps": 226.40398773185728, + "prefill_tps": 644.48145081118, + "itl_mean_ms": 4.416883333275716, + "itl_p50_ms": 4.611999727785587, + "itl_p95_ms": 5.033666733652353, + "round_gap_p50_ms": 4.6632918529212475, + "round_gap_p95_ms": 5.0445422530174255, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021457749884575605, + "decode_tps": 214.62685536293307, + "prefill_tps": 2516.5732796063867, + "itl_mean_ms": 4.659249180672216, + "itl_p50_ms": 4.624708089977503, + "itl_p95_ms": 5.02549996599555, + "round_gap_p50_ms": 4.665624815970659, + "round_gap_p95_ms": 5.025624763220549, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06293591717258096, + "decode_tps": 222.3952286815589, + "prefill_tps": 858.0156201096243, + "itl_mean_ms": 4.496499344560445, + "itl_p50_ms": 4.6320422552526, + "itl_p95_ms": 5.02470787614584, + "round_gap_p50_ms": 4.659791011363268, + "round_gap_p95_ms": 5.028042010962963, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.042323541827499866, + "decode_tps": 218.44678128584536, + "prefill_tps": 1275.885657681733, + "itl_mean_ms": 4.5777740194370935, + "itl_p50_ms": 4.626791924238205, + "itl_p95_ms": 5.034125410020351, + "round_gap_p50_ms": 4.6547092497348785, + "round_gap_p95_ms": 5.0383745692670345, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.5376014694863676, + "itl_p50_ms": 4.624708089977503, + "itl_p95_ms": 5.02549996599555, + "round_gap_p50_ms": 4.6632918529212475, + "round_gap_p95_ms": 5.033666733652353, + "round_gap_mean_ms": 4.902916842029762, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2576.3384256292893 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 859.3933484056931, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2087285001762211, + "end_to_end_tps": 847.171221536276, + "timing_valid": true, + "steady_tps": 906.0121544619261, + "per_stream_tps": [ + 222.6, + 214.9, + 226.7, + 218.7 + ], + "ttft_s": [ + 0.06, + 0.02, + 0.08, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.06286758417263627, + "decode_tps": 222.63901976942705, + "prefill_tps": 858.9482276225914, + "itl_mean_ms": 4.491575650286441, + "itl_p50_ms": 4.6135419979691505, + "itl_p95_ms": 5.026792176067829, + "round_gap_p50_ms": 4.643499851226807, + "round_gap_p95_ms": 5.033375229686499, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021316375117748976, + "decode_tps": 214.85508770342093, + "prefill_tps": 2533.2637327740194, + "itl_mean_ms": 4.654299838504955, + "itl_p50_ms": 4.616333171725273, + "itl_p95_ms": 5.029041785746813, + "round_gap_p50_ms": 4.647457972168922, + "round_gap_p95_ms": 5.031208973377943, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08356091706082225, + "decode_tps": 226.74831305879255, + "prefill_tps": 646.2351288066229, + "itl_mean_ms": 4.410176139836218, + "itl_p50_ms": 4.61312523111701, + "itl_p95_ms": 5.0145830027759075, + "round_gap_p50_ms": 4.636333789676428, + "round_gap_p95_ms": 5.019790958613157, + "tokens_per_chunk": 1.080168776371308 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04216687520965934, + "decode_tps": 218.68075345864906, + "prefill_tps": 1280.626077495778, + "itl_mean_ms": 4.572876141059632, + "itl_p50_ms": 4.61520766839385, + "itl_p95_ms": 5.024875048547983, + "round_gap_p50_ms": 4.651082679629326, + "round_gap_p95_ms": 5.031832959502935, + "tokens_per_chunk": 1.080168776371308 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.532231942421811, + "itl_p50_ms": 4.61520766839385, + "itl_p95_ms": 5.020833108574152, + "round_gap_p50_ms": 4.64604189619422, + "round_gap_p95_ms": 5.029041785746813, + "round_gap_mean_ms": 4.897115022532042, + "tokens_per_chunk": 1.080168776371308, + "aggregate_prefill_tps": 2581.308213291005 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db", + "complete": true, + "error": null + } + } + ] + } + ] + } + ] + } + }, + { + "name": "vl-4-0-old.concurrent.json", + "data": { + "label": "vl-4-0-old", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 301.7, + "per_stream_tps": 301.7, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 301.709, + "prefill_tps": 2508.574, + "itl_mean_ms": 3.314, + "itl_p50_ms": 3.36, + "itl_p95_ms": 3.87, + "round_gap_p50_ms": 3.368, + "round_gap_p95_ms": 3.87, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2508.574, + "itl_mean_ms": 3.314, + "itl_p50_ms": 3.36, + "itl_p95_ms": 3.87, + "round_gap_p50_ms": 3.368, + "round_gap_p95_ms": 3.87, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 295.49816898478065, + "makespan_s": 0.866333625279367, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 301.7540640926198, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8662996659986675, + "end_to_end_tps": 295.5097526268627, + "timing_valid": true, + "per_stream_tps": [ + 301.8 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021180750336498022, + "decode_tps": 301.7540640926198, + "prefill_tps": 2502.2720705352926, + "itl_mean_ms": 3.3139570232699898, + "itl_p50_ms": 3.3591659739613533, + "itl_p95_ms": 3.8580000400543213, + "round_gap_p50_ms": 3.367666620761156, + "round_gap_p95_ms": 3.8580000400543213, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.3139570232699898, + "itl_p50_ms": 3.3591659739613533, + "itl_p95_ms": 3.8580000400543213, + "round_gap_p50_ms": 3.367666620761156, + "round_gap_p95_ms": 3.8580000400543213, + "round_gap_mean_ms": 3.3938114093728813, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2502.2720705352926 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 301.7089147514105, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.866333625279367, + "end_to_end_tps": 295.49816898478065, + "timing_valid": true, + "per_stream_tps": [ + 301.7 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02109108306467533, + "decode_tps": 301.7089147514105, + "prefill_tps": 2512.9103060983966, + "itl_mean_ms": 3.3144529415842365, + "itl_p50_ms": 3.3653751015663147, + "itl_p95_ms": 3.870416898280382, + "round_gap_p50_ms": 3.38225020095706, + "round_gap_p95_ms": 3.870416898280382, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.3144529415842365, + "itl_p50_ms": 3.3653751015663147, + "itl_p95_ms": 3.870416898280382, + "round_gap_p50_ms": 3.38225020095706, + "round_gap_p95_ms": 3.870416898280382, + "round_gap_mean_ms": 3.394319277526025, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2512.9103060983966 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 301.364427514273, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8673348752781749, + "end_to_end_tps": 295.15704636907947, + "timing_valid": true, + "per_stream_tps": [ + 301.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021127541083842516, + "decode_tps": 301.364427514273, + "prefill_tps": 2508.573988315765, + "itl_mean_ms": 3.3182416659067657, + "itl_p50_ms": 3.3597080036997795, + "itl_p95_ms": 3.873333800584078, + "round_gap_p50_ms": 3.3679171465337276, + "round_gap_p95_ms": 3.873333800584078, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.3182416659067657, + "itl_p50_ms": 3.3597080036997795, + "itl_p95_ms": 3.873333800584078, + "round_gap_p50_ms": 3.3679171465337276, + "round_gap_p95_ms": 3.873333800584078, + "round_gap_mean_ms": 3.398199296410543, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2508.573988315765 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 814.1, + "per_stream_tps": 208.8, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.052, + "decode_tps": 208.793, + "prefill_tps": 1055.518, + "itl_mean_ms": 4.79, + "itl_p50_ms": 4.725, + "itl_p95_ms": 5.167, + "round_gap_p50_ms": 4.733, + "round_gap_p95_ms": 5.167, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2548.367, + "itl_mean_ms": 4.792, + "itl_p50_ms": 4.726, + "itl_p95_ms": 5.165, + "round_gap_p50_ms": 4.735, + "round_gap_p95_ms": 5.165, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 803.3194719200513, + "makespan_s": 1.2747107916511595, + "steady_tps": 853.8, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 814.1348438522192, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2747107916511595, + "end_to_end_tps": 803.3194719200513, + "timing_valid": true, + "steady_tps": 853.7949753961888, + "per_stream_tps": [ + 206.9, + 210.5, + 214.1, + 203.5 + ], + "ttft_s": [ + 0.04, + 0.06, + 0.08, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.04190954100340605, + "decode_tps": 206.9448107502717, + "prefill_tps": 1264.6285006006774, + "itl_mean_ms": 4.832206211765023, + "itl_p50_ms": 4.716292023658752, + "itl_p95_ms": 5.175999831408262, + "round_gap_p50_ms": 4.716666880995035, + "round_gap_p95_ms": 5.175999831408262, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06250687502324581, + "decode_tps": 210.46800436317056, + "prefill_tps": 847.9067299443417, + "itl_mean_ms": 4.7513160160651395, + "itl_p50_ms": 4.723624791949987, + "itl_p95_ms": 5.16587495803833, + "round_gap_p50_ms": 4.728875122964382, + "round_gap_p95_ms": 5.16587495803833, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08305587526410818, + "decode_tps": 214.10861731957877, + "prefill_tps": 638.124633946317, + "itl_mean_ms": 4.670526635120897, + "itl_p50_ms": 4.703917074948549, + "itl_p95_ms": 5.172249861061573, + "round_gap_p50_ms": 4.724165890365839, + "round_gap_p95_ms": 5.172249861061573, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.02131849993020296, + "decode_tps": 203.54017559440442, + "prefill_tps": 2486.103627062067, + "itl_mean_ms": 4.913034967566822, + "itl_p50_ms": 4.72299987450242, + "itl_p95_ms": 5.166333634406328, + "round_gap_p50_ms": 4.72933379933238, + "round_gap_p95_ms": 5.166333634406328, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.7917709576294705, + "itl_p50_ms": 4.716666880995035, + "itl_p95_ms": 5.166459362953901, + "round_gap_p50_ms": 4.725332837551832, + "round_gap_p95_ms": 5.167459137737751, + "round_gap_mean_ms": 4.88760637678206, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2548.1407228271746 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 813.6843894432064, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2748617082834244, + "end_to_end_tps": 803.2243759041091, + "timing_valid": true, + "steady_tps": 853.651618227856, + "per_stream_tps": [ + 206.9, + 203.5, + 214.0, + 210.4 + ], + "ttft_s": [ + 0.04, + 0.02, + 0.08, + 0.06 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.04190954193472862, + "decode_tps": 206.9175094185152, + "prefill_tps": 1264.6284724978393, + "itl_mean_ms": 4.832843787895114, + "itl_p50_ms": 4.727332852780819, + "itl_p95_ms": 5.170958116650581, + "round_gap_p50_ms": 4.740333184599876, + "round_gap_p95_ms": 5.170958116650581, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.02120312489569187, + "decode_tps": 203.50088001959082, + "prefill_tps": 2499.6315524589836, + "itl_mean_ms": 4.913983663872761, + "itl_p50_ms": 4.733624868094921, + "itl_p95_ms": 5.16508286818862, + "round_gap_p50_ms": 4.736457951366901, + "round_gap_p95_ms": 5.16508286818862, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08304875018075109, + "decode_tps": 213.98906663958724, + "prefill_tps": 638.1793812025875, + "itl_mean_ms": 4.673135948970037, + "itl_p50_ms": 4.725832957774401, + "itl_p95_ms": 5.171833094209433, + "round_gap_p50_ms": 4.732375033199787, + "round_gap_p95_ms": 5.171833094209433, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06259462516754866, + "decode_tps": 210.43271066963496, + "prefill_tps": 846.7180665773383, + "itl_mean_ms": 4.752112904965293, + "itl_p50_ms": 4.719417076557875, + "itl_p95_ms": 5.165334325283766, + "round_gap_p50_ms": 4.730249755084515, + "round_gap_p95_ms": 5.165334325283766, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.793019076425801, + "itl_p50_ms": 4.725832957774401, + "itl_p95_ms": 5.165250040590763, + "round_gap_p50_ms": 4.736040718853474, + "round_gap_p95_ms": 5.165334325283766, + "round_gap_mean_ms": 4.888879457954317, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2548.366639015444 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 814.9516464163819, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.273495374713093, + "end_to_end_tps": 804.0861555784589, + "timing_valid": true, + "steady_tps": 854.647091215278, + "per_stream_tps": [ + 203.7, + 214.3, + 210.7, + 207.2 + ], + "ttft_s": [ + 0.02, + 0.08, + 0.06, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021315874997526407, + "decode_tps": 203.74160142531468, + "prefill_tps": 2486.40977703943, + "itl_mean_ms": 4.908177775203013, + "itl_p50_ms": 4.730584099888802, + "itl_p95_ms": 5.167499650269747, + "round_gap_p50_ms": 4.7339582815766335, + "round_gap_p95_ms": 5.167499650269747, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08311354229226708, + "decode_tps": 214.32641836297898, + "prefill_tps": 637.6818811743889, + "itl_mean_ms": 4.665780390667565, + "itl_p50_ms": 4.720957949757576, + "itl_p95_ms": 5.161790642887354, + "round_gap_p50_ms": 4.7382088378071785, + "round_gap_p95_ms": 5.161790642887354, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06245420780032873, + "decode_tps": 210.67642004144145, + "prefill_tps": 848.6217641162848, + "itl_mean_ms": 4.746615685814736, + "itl_p50_ms": 4.728750325739384, + "itl_p95_ms": 5.156124942004681, + "round_gap_p50_ms": 4.734707996249199, + "round_gap_p95_ms": 5.156124942004681, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.041983041912317276, + "decode_tps": 207.1536168499669, + "prefill_tps": 1262.414479415092, + "itl_mean_ms": 4.827335458614078, + "itl_p50_ms": 4.7288332134485245, + "itl_p95_ms": 5.1695420406758785, + "round_gap_p50_ms": 4.7340840101242065, + "round_gap_p95_ms": 5.1695420406758785, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.786977327574848, + "itl_p50_ms": 4.728750325739384, + "itl_p95_ms": 5.1644593477249146, + "round_gap_p50_ms": 4.734707996249199, + "round_gap_p95_ms": 5.164791829884052, + "round_gap_mean_ms": 4.882716874126345, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2548.4968096889543 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 299.5, + "per_stream_tps": 299.5, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 299.526, + "prefill_tps": 2530.291, + "itl_mean_ms": 3.339, + "itl_p50_ms": 3.404, + "itl_p95_ms": 3.87, + "round_gap_p50_ms": 3.417, + "round_gap_p95_ms": 3.87, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2530.291, + "itl_mean_ms": 3.339, + "itl_p50_ms": 3.404, + "itl_p95_ms": 3.87, + "round_gap_p50_ms": 3.417, + "round_gap_p95_ms": 3.87, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 293.3290136664728, + "makespan_s": 0.8727401248179376, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 299.36909444641225, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8731484999880195, + "end_to_end_tps": 293.1918224717933, + "timing_valid": true, + "per_stream_tps": [ + 299.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02130099991336465, + "decode_tps": 299.36909444641225, + "prefill_tps": 2535.092259500897, + "itl_mean_ms": 3.3403581683980486, + "itl_p50_ms": 3.40087478980422, + "itl_p95_ms": 3.8569169119000435, + "round_gap_p50_ms": 3.41595895588398, + "round_gap_p95_ms": 3.8569169119000435, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.3403581683980486, + "itl_p50_ms": 3.40087478980422, + "itl_p95_ms": 3.8569169119000435, + "round_gap_p50_ms": 3.41595895588398, + "round_gap_p95_ms": 3.8569169119000435, + "round_gap_mean_ms": 3.4208487266727, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2535.092259500897 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 299.525501686138, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8727401248179376, + "end_to_end_tps": 293.3290136664728, + "timing_valid": true, + "per_stream_tps": [ + 299.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021341416984796524, + "decode_tps": 299.525501686138, + "prefill_tps": 2530.2912191102037, + "itl_mean_ms": 3.338613888869683, + "itl_p50_ms": 3.404832910746336, + "itl_p95_ms": 3.87366721406579, + "round_gap_p50_ms": 3.4183328971266747, + "round_gap_p95_ms": 3.87366721406579, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.338613888869683, + "itl_p50_ms": 3.404832910746336, + "itl_p95_ms": 3.87366721406579, + "round_gap_p50_ms": 3.4183328971266747, + "round_gap_p95_ms": 3.87366721406579, + "round_gap_mean_ms": 3.4190624163123258, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2530.2912191102037 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 299.6400057055273, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8724502911791205, + "end_to_end_tps": 293.4264594651173, + "timing_valid": true, + "per_stream_tps": [ + 299.6 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021375915966928005, + "decode_tps": 299.6400057055273, + "prefill_tps": 2526.207535786851, + "itl_mean_ms": 3.3373380755530855, + "itl_p50_ms": 3.4039998427033424, + "itl_p95_ms": 3.87045880779624, + "round_gap_p50_ms": 3.4170839935541153, + "round_gap_p95_ms": 3.87045880779624, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.3373380755530855, + "itl_p50_ms": 3.4039998427033424, + "itl_p95_ms": 3.87045880779624, + "round_gap_p50_ms": 3.4170839935541153, + "round_gap_p95_ms": 3.87045880779624, + "round_gap_mean_ms": 3.4177558605061718, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2526.207535786851 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 810.8, + "per_stream_tps": 207.9, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.053, + "decode_tps": 207.908, + "prefill_tps": 1061.939, + "itl_mean_ms": 4.81, + "itl_p50_ms": 4.762, + "itl_p95_ms": 5.167, + "round_gap_p50_ms": 4.764, + "round_gap_p95_ms": 5.167, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2562.913, + "itl_mean_ms": 4.81, + "itl_p50_ms": 4.762, + "itl_p95_ms": 5.167, + "round_gap_p50_ms": 4.766, + "round_gap_p95_ms": 5.167, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 800.157530886016, + "makespan_s": 1.2797480002045631, + "steady_tps": 851.5, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 811.1227998779179, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2792930831201375, + "end_to_end_tps": 800.4420671942592, + "timing_valid": true, + "steady_tps": 851.8511713913853, + "per_stream_tps": [ + 206.3, + 213.4, + 209.8, + 202.8 + ], + "ttft_s": [ + 0.04, + 0.08, + 0.06, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.04242616659030318, + "decode_tps": 206.26284078802743, + "prefill_tps": 1272.7994146033006, + "itl_mean_ms": 4.84818300853173, + "itl_p50_ms": 4.75708395242691, + "itl_p95_ms": 5.1666246727108955, + "round_gap_p50_ms": 4.763707984238863, + "round_gap_p95_ms": 5.1666246727108955, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08415862498804927, + "decode_tps": 213.42819781920804, + "prefill_tps": 641.645464236947, + "itl_mean_ms": 4.685416501745874, + "itl_p50_ms": 4.763834178447723, + "itl_p95_ms": 5.1675839349627495, + "round_gap_p50_ms": 4.765749908983707, + "round_gap_p95_ms": 5.1675839349627495, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06329445913434029, + "decode_tps": 209.80806063624925, + "prefill_tps": 853.1552483193968, + "itl_mean_ms": 4.7662611101187915, + "itl_p50_ms": 4.761583171784878, + "itl_p95_ms": 5.163792055100203, + "round_gap_p50_ms": 4.767249803990126, + "round_gap_p95_ms": 5.163792055100203, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021528958342969418, + "decode_tps": 202.83073484217724, + "prefill_tps": 2508.249546482794, + "itl_mean_ms": 4.930219282487444, + "itl_p50_ms": 4.752458073198795, + "itl_p95_ms": 5.168457981199026, + "round_gap_p50_ms": 4.763041622936726, + "round_gap_p95_ms": 5.168457981199026, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.807519975720959, + "itl_p50_ms": 4.7599999234080315, + "itl_p95_ms": 5.1666246727108955, + "round_gap_p50_ms": 4.765749908983707, + "round_gap_p95_ms": 5.167416762560606, + "round_gap_mean_ms": 4.903670375235379, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2563.376056673976 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 810.8225734210974, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2797480002045631, + "end_to_end_tps": 800.157530886016, + "timing_valid": true, + "steady_tps": 851.4997576520444, + "per_stream_tps": [ + 202.8, + 206.2, + 213.3, + 209.7 + ], + "ttft_s": [ + 0.02, + 0.04, + 0.08, + 0.06 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02157979179173708, + "decode_tps": 202.7638904721657, + "prefill_tps": 2502.341103248116, + "itl_mean_ms": 4.931844608383436, + "itl_p50_ms": 4.762374795973301, + "itl_p95_ms": 5.162958055734634, + "round_gap_p50_ms": 4.763958975672722, + "round_gap_p95_ms": 5.162958055734634, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.042455459013581276, + "decode_tps": 206.18721328317525, + "prefill_tps": 1271.9212382729318, + "itl_mean_ms": 4.8499612758557005, + "itl_p50_ms": 4.765999969094992, + "itl_p95_ms": 5.1644169725477695, + "round_gap_p50_ms": 4.7707087360322475, + "round_gap_p95_ms": 5.1644169725477695, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08416950004175305, + "decode_tps": 213.33104296049535, + "prefill_tps": 641.562560942061, + "itl_mean_ms": 4.68755032611536, + "itl_p50_ms": 4.763042088598013, + "itl_p95_ms": 5.156666971743107, + "round_gap_p50_ms": 4.763582721352577, + "round_gap_p95_ms": 5.156666971743107, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06340754218399525, + "decode_tps": 209.69372277573757, + "prefill_tps": 851.6337038156037, + "itl_mean_ms": 4.768859967589378, + "itl_p50_ms": 4.766208119690418, + "itl_p95_ms": 5.159999709576368, + "round_gap_p50_ms": 4.772082902491093, + "round_gap_p95_ms": 5.159999709576368, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.809554044485969, + "itl_p50_ms": 4.763042088598013, + "itl_p95_ms": 5.159999709576368, + "round_gap_p50_ms": 4.7707087360322475, + "round_gap_p95_ms": 5.160292144864798, + "round_gap_mean_ms": 4.905745125375688, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2562.913471053916 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 810.3409295004235, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2808924997225404, + "end_to_end_tps": 799.4425763456442, + "timing_valid": true, + "steady_tps": 850.7649268526958, + "per_stream_tps": [ + 202.6, + 209.6, + 206.0, + 213.2 + ], + "ttft_s": [ + 0.02, + 0.06, + 0.04, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02163279103115201, + "decode_tps": 202.59002060594403, + "prefill_tps": 2496.2104946254058, + "itl_mean_ms": 4.936077290525039, + "itl_p50_ms": 4.762541968375444, + "itl_p95_ms": 5.173000041395426, + "round_gap_p50_ms": 4.763499833643436, + "round_gap_p95_ms": 5.173000041395426, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06344679184257984, + "decode_tps": 209.55300082520802, + "prefill_tps": 851.1068634326126, + "itl_mean_ms": 4.772062418872819, + "itl_p50_ms": 4.758375231176615, + "itl_p95_ms": 5.178000312298536, + "round_gap_p50_ms": 4.759666975587606, + "round_gap_p95_ms": 5.178000312298536, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.042495499830693007, + "decode_tps": 206.01371547123458, + "prefill_tps": 1270.7227874749622, + "itl_mean_ms": 4.854045749879351, + "itl_p50_ms": 4.763874690979719, + "itl_p95_ms": 5.182374734431505, + "round_gap_p50_ms": 4.767999984323978, + "round_gap_p95_ms": 5.182374734431505, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08437824994325638, + "decode_tps": 213.2060970264828, + "prefill_tps": 639.9753495280421, + "itl_mean_ms": 4.690297388051655, + "itl_p50_ms": 4.761500284075737, + "itl_p95_ms": 5.176540929824114, + "round_gap_p50_ms": 4.7638751566410065, + "round_gap_p95_ms": 5.176540929824114, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.813120711832216, + "itl_p50_ms": 4.761500284075737, + "itl_p95_ms": 5.175875034183264, + "round_gap_p50_ms": 4.765209276229143, + "round_gap_p95_ms": 5.176540929824114, + "round_gap_mean_ms": 4.90938312606886, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2559.9013981121684 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + } + ] + } + ] + } + }, + { + "name": "vl-4-1-new.concurrent.json", + "data": { + "label": "vl-4-1-new", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 305.5, + "per_stream_tps": 305.5, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 305.471, + "prefill_tps": 2518.189, + "itl_mean_ms": 3.274, + "itl_p50_ms": 3.325, + "itl_p95_ms": 3.822, + "round_gap_p50_ms": 3.335, + "round_gap_p95_ms": 3.822, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2518.189, + "itl_mean_ms": 3.274, + "itl_p50_ms": 3.325, + "itl_p95_ms": 3.822, + "round_gap_p50_ms": 3.335, + "round_gap_p95_ms": 3.822, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 299.1173701464553, + "makespan_s": 0.8558513331227005, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 305.6852512007681, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8552945419214666, + "end_to_end_tps": 299.312093615004, + "timing_valid": true, + "per_stream_tps": [ + 305.7 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021046875044703484, + "decode_tps": 305.6852512007681, + "prefill_tps": 2518.188561837717, + "itl_mean_ms": 3.2713387252799437, + "itl_p50_ms": 3.3248751424252987, + "itl_p95_ms": 3.8135419599711895, + "round_gap_p50_ms": 3.335000015795231, + "round_gap_p95_ms": 3.8135419599711895, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.2713387252799437, + "itl_p50_ms": 3.3248751424252987, + "itl_p95_ms": 3.8135419599711895, + "round_gap_p50_ms": 3.335000015795231, + "round_gap_p95_ms": 3.8135419599711895, + "round_gap_mean_ms": 3.3501661644433156, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2518.188561837717 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 304.87353805856026, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.857544083148241, + "end_to_end_tps": 298.52692710579413, + "timing_valid": true, + "per_stream_tps": [ + 304.9 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02106691710650921, + "decode_tps": 304.87353805856026, + "prefill_tps": 2515.792877146898, + "itl_mean_ms": 3.280048528868778, + "itl_p50_ms": 3.3246250823140144, + "itl_p95_ms": 3.8298331201076508, + "round_gap_p50_ms": 3.3371662721037865, + "round_gap_p95_ms": 3.8298331201076508, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.280048528868778, + "itl_p50_ms": 3.3246250823140144, + "itl_p95_ms": 3.8298331201076508, + "round_gap_p50_ms": 3.3371662721037865, + "round_gap_p95_ms": 3.8298331201076508, + "round_gap_mean_ms": 3.3590858428174233, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2515.792877146898 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 305.4710310974011, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8558513331227005, + "end_to_end_tps": 299.1173701464553, + "timing_valid": true, + "per_stream_tps": [ + 305.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021024875342845917, + "decode_tps": 305.4710310974011, + "prefill_tps": 2520.8235071906947, + "itl_mean_ms": 3.2736328430473804, + "itl_p50_ms": 3.328041173517704, + "itl_p95_ms": 3.821916878223419, + "round_gap_p50_ms": 3.332667052745819, + "round_gap_p95_ms": 3.821916878223419, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.2736328430473804, + "itl_p50_ms": 3.328041173517704, + "itl_p95_ms": 3.821916878223419, + "round_gap_p50_ms": 3.332667052745819, + "round_gap_p95_ms": 3.821916878223419, + "round_gap_mean_ms": 3.352515562156956, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2520.8235071906947 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 824.5, + "per_stream_tps": 211.6, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.052, + "decode_tps": 211.517, + "prefill_tps": 1054.559, + "itl_mean_ms": 4.728, + "itl_p50_ms": 4.656, + "itl_p95_ms": 5.102, + "round_gap_p50_ms": 4.666, + "round_gap_p95_ms": 5.102, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2550.338, + "itl_mean_ms": 4.729, + "itl_p50_ms": 4.656, + "itl_p95_ms": 5.103, + "round_gap_p50_ms": 4.668, + "round_gap_p95_ms": 5.104, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 813.6214590360078, + "makespan_s": 1.2585705411620438, + "steady_tps": 865.4, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 824.224135505757, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2593117500655353, + "end_to_end_tps": 813.1425756542893, + "timing_valid": true, + "steady_tps": 864.8813684427847, + "per_stream_tps": [ + 209.5, + 206.1, + 216.9, + 213.2 + ], + "ttft_s": [ + 0.04, + 0.02, + 0.08, + 0.06 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.04186008311808109, + "decode_tps": 209.54855954147118, + "prefill_tps": 1266.1226651293275, + "itl_mean_ms": 4.772163560504422, + "itl_p50_ms": 4.660207778215408, + "itl_p95_ms": 5.105082876980305, + "round_gap_p50_ms": 4.666666965931654, + "round_gap_p95_ms": 5.105082876980305, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021223625168204308, + "decode_tps": 206.06390860584838, + "prefill_tps": 2497.2171144164736, + "itl_mean_ms": 4.852863399348423, + "itl_p50_ms": 4.6646250411868095, + "itl_p95_ms": 5.101833958178759, + "round_gap_p50_ms": 4.677499644458294, + "round_gap_p95_ms": 5.101833958178759, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08297299966216087, + "decode_tps": 216.88730630912056, + "prefill_tps": 638.7620095187447, + "itl_mean_ms": 4.6106893806626985, + "itl_p50_ms": 4.64849965646863, + "itl_p95_ms": 5.106708034873009, + "round_gap_p50_ms": 4.654749762266874, + "round_gap_p95_ms": 5.106708034873009, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.062496041879057884, + "decode_tps": 213.15460663592262, + "prefill_tps": 848.0537071862152, + "itl_mean_ms": 4.69143039309511, + "itl_p50_ms": 4.660916049033403, + "itl_p95_ms": 5.106083117425442, + "round_gap_p50_ms": 4.677874967455864, + "round_gap_p95_ms": 5.106083117425442, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.731786683402663, + "itl_p50_ms": 4.657125100493431, + "itl_p95_ms": 5.105082876980305, + "round_gap_p50_ms": 4.671500064432621, + "round_gap_p95_ms": 5.105374846607447, + "round_gap_mean_ms": 4.826422417070717, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2551.254499591046 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 825.1571262681787, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2576085408218205, + "end_to_end_tps": 814.2438340398338, + "timing_valid": true, + "steady_tps": 866.2736064061186, + "per_stream_tps": [ + 213.5, + 217.1, + 206.4, + 209.9 + ], + "ttft_s": [ + 0.06, + 0.08, + 0.02, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.06261241715401411, + "decode_tps": 213.4985432394523, + "prefill_tps": 846.4774626034725, + "itl_mean_ms": 4.683872708575983, + "itl_p50_ms": 4.654749762266874, + "itl_p95_ms": 5.08833397179842, + "round_gap_p50_ms": 4.6681249514222145, + "round_gap_p95_ms": 5.08833397179842, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08316183416172862, + "decode_tps": 217.14233744120943, + "prefill_tps": 637.3115808981374, + "itl_mean_ms": 4.605274179986879, + "itl_p50_ms": 4.646959248930216, + "itl_p95_ms": 5.086000077426434, + "round_gap_p50_ms": 4.651833325624466, + "round_gap_p95_ms": 5.086000077426434, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.02133604185655713, + "decode_tps": 206.38209649914984, + "prefill_tps": 2484.0596187578108, + "itl_mean_ms": 4.845381537269728, + "itl_p50_ms": 4.651166964322329, + "itl_p95_ms": 5.093665793538094, + "round_gap_p50_ms": 4.666333086788654, + "round_gap_p95_ms": 5.093665793538094, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04205716587603092, + "decode_tps": 209.88007714925564, + "prefill_tps": 1260.1895276591993, + "itl_mean_ms": 4.764625654720208, + "itl_p50_ms": 4.668791778385639, + "itl_p95_ms": 5.088582634925842, + "round_gap_p50_ms": 4.6697501093149185, + "round_gap_p95_ms": 5.088582634925842, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.724788520138199, + "itl_p50_ms": 4.651833325624466, + "itl_p95_ms": 5.087457597255707, + "round_gap_p50_ms": 4.668166860938072, + "round_gap_p95_ms": 5.08833397179842, + "round_gap_mean_ms": 4.819284290540963, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2546.902197880832 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 824.5112648069562, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2585705411620438, + "end_to_end_tps": 813.6214590360078, + "timing_valid": true, + "steady_tps": 865.4434597941244, + "per_stream_tps": [ + 209.7, + 217.0, + 213.3, + 206.2 + ], + "ttft_s": [ + 0.04, + 0.08, + 0.06, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.04188804188743234, + "decode_tps": 209.68319022928569, + "prefill_tps": 1265.277573547824, + "itl_mean_ms": 4.76909951106006, + "itl_p50_ms": 4.657791927456856, + "itl_p95_ms": 5.105666816234589, + "round_gap_p50_ms": 4.665832966566086, + "round_gap_p95_ms": 5.105666816234589, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08304858393967152, + "decode_tps": 216.9681424670553, + "prefill_tps": 638.1806586672263, + "itl_mean_ms": 4.608971568956679, + "itl_p50_ms": 4.653790965676308, + "itl_p95_ms": 5.10270893573761, + "round_gap_p50_ms": 4.6591246500611305, + "round_gap_p95_ms": 5.10270893573761, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06243158411234617, + "decode_tps": 213.29739164402793, + "prefill_tps": 848.9292840083322, + "itl_mean_ms": 4.688289867458391, + "itl_p50_ms": 4.651166032999754, + "itl_p95_ms": 5.1007079891860485, + "round_gap_p50_ms": 4.666333086788654, + "round_gap_p95_ms": 5.1007079891860485, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.02131750015541911, + "decode_tps": 206.1899573802127, + "prefill_tps": 2486.220223459311, + "itl_mean_ms": 4.849896729722912, + "itl_p50_ms": 4.656707867980003, + "itl_p95_ms": 5.104625131934881, + "round_gap_p50_ms": 4.660374950617552, + "round_gap_p95_ms": 5.104625131934881, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.72906441929951, + "itl_p50_ms": 4.6559590846300125, + "itl_p95_ms": 5.10270893573761, + "round_gap_p50_ms": 4.665832966566086, + "round_gap_p95_ms": 5.103542003780603, + "round_gap_mean_ms": 4.8236457076855, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2550.337583116069 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 303.0, + "per_stream_tps": 303.0, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 302.962, + "prefill_tps": 2535.355, + "itl_mean_ms": 3.301, + "itl_p50_ms": 3.36, + "itl_p95_ms": 3.839, + "round_gap_p50_ms": 3.379, + "round_gap_p95_ms": 3.839, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2535.355, + "itl_mean_ms": 3.301, + "itl_p50_ms": 3.36, + "itl_p95_ms": 3.839, + "round_gap_p50_ms": 3.379, + "round_gap_p95_ms": 3.839, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 296.6264819980816, + "makespan_s": 0.8630382502451539, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 302.96219839482046, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8630382502451539, + "end_to_end_tps": 296.6264819980816, + "timing_valid": true, + "per_stream_tps": [ + 303.0 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021298792213201523, + "decode_tps": 302.96219839482046, + "prefill_tps": 2535.3550313772935, + "itl_mean_ms": 3.3007418262023553, + "itl_p50_ms": 3.3597913570702076, + "itl_p95_ms": 3.839458804577589, + "round_gap_p50_ms": 3.372624982148409, + "round_gap_p95_ms": 3.839458804577589, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.3007418262023553, + "itl_p50_ms": 3.3597913570702076, + "itl_p95_ms": 3.839458804577589, + "round_gap_p50_ms": 3.372624982148409, + "round_gap_p95_ms": 3.839458804577589, + "round_gap_mean_ms": 3.380277773821689, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2535.3550313772935 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 303.07128595927406, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8627682500518858, + "end_to_end_tps": 296.719310179303, + "timing_valid": true, + "per_stream_tps": [ + 303.1 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021329583134502172, + "decode_tps": 303.07128595927406, + "prefill_tps": 2531.695048116108, + "itl_mean_ms": 3.299553756255145, + "itl_p50_ms": 3.3582081086933613, + "itl_p95_ms": 3.8338331505656242, + "round_gap_p50_ms": 3.3794171176850796, + "round_gap_p95_ms": 3.8338331505656242, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.299553756255145, + "itl_p50_ms": 3.3582081086933613, + "itl_p95_ms": 3.8338331505656242, + "round_gap_p50_ms": 3.3794171176850796, + "round_gap_p95_ms": 3.8338331505656242, + "round_gap_mean_ms": 3.37906107568298, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2531.695048116108 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 302.86168661087646, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8633011253550649, + "end_to_end_tps": 296.53615926274904, + "timing_valid": true, + "per_stream_tps": [ + 302.9 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02128316694870591, + "decode_tps": 302.86168661087646, + "prefill_tps": 2537.2163893721363, + "itl_mean_ms": 3.301837255119109, + "itl_p50_ms": 3.3766250126063824, + "itl_p95_ms": 3.840333316475153, + "round_gap_p50_ms": 3.384375013411045, + "round_gap_p95_ms": 3.840333316475153, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.301837255119109, + "itl_p50_ms": 3.3766250126063824, + "itl_p95_ms": 3.840333316475153, + "round_gap_p50_ms": 3.384375013411045, + "round_gap_p95_ms": 3.840333316475153, + "round_gap_mean_ms": 3.3813995986159546, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2537.2163893721363 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 821.7, + "per_stream_tps": 210.9, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.053, + "decode_tps": 210.829, + "prefill_tps": 1061.804, + "itl_mean_ms": 4.743, + "itl_p50_ms": 4.687, + "itl_p95_ms": 5.105, + "round_gap_p50_ms": 4.698, + "round_gap_p95_ms": 5.105, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2564.295, + "itl_mean_ms": 4.745, + "itl_p50_ms": 4.687, + "itl_p95_ms": 5.107, + "round_gap_p50_ms": 4.701, + "round_gap_p95_ms": 5.107, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 810.4254696466818, + "makespan_s": 1.2635338329710066, + "steady_tps": 863.2, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 823.021525693088, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2613546666689217, + "end_to_end_tps": 811.8255928003617, + "timing_valid": true, + "steady_tps": 864.7186838957481, + "per_stream_tps": [ + 212.9, + 216.7, + 209.3, + 205.8 + ], + "ttft_s": [ + 0.06, + 0.08, + 0.04, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.06325083319097757, + "decode_tps": 212.93182271178225, + "prefill_tps": 853.7436943629517, + "itl_mean_ms": 4.696338890376044, + "itl_p50_ms": 4.683167207986116, + "itl_p95_ms": 5.092000123113394, + "round_gap_p50_ms": 4.700249992311001, + "round_gap_p95_ms": 5.092000123113394, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08401883393526077, + "decode_tps": 216.70263916910613, + "prefill_tps": 642.7130379077713, + "itl_mean_ms": 4.614618464427836, + "itl_p50_ms": 4.68500005081296, + "itl_p95_ms": 5.0965421833097935, + "round_gap_p50_ms": 4.690374713391066, + "round_gap_p95_ms": 5.0965421833097935, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04232979193329811, + "decode_tps": 209.28036682019072, + "prefill_tps": 1275.6972697879407, + "itl_mean_ms": 4.778279086538389, + "itl_p50_ms": 4.694540984928608, + "itl_p95_ms": 5.104334093630314, + "round_gap_p50_ms": 4.709708038717508, + "round_gap_p95_ms": 5.104334093630314, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.02153441682457924, + "decode_tps": 205.76054899133425, + "prefill_tps": 2507.613762652015, + "itl_mean_ms": 4.860018137111967, + "itl_p50_ms": 4.687707871198654, + "itl_p95_ms": 5.104041192680597, + "round_gap_p50_ms": 4.6966662630438805, + "round_gap_p95_ms": 5.104041192680597, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.737313644613559, + "itl_p50_ms": 4.686624743044376, + "itl_p95_ms": 5.09816687554121, + "round_gap_p50_ms": 4.700666759163141, + "round_gap_p95_ms": 5.098334047943354, + "round_gap_mean_ms": 4.8320599175058305, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2567.0749765264095 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 821.6581848173277, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2635338329710066, + "end_to_end_tps": 810.4254696466818, + "timing_valid": true, + "steady_tps": 863.199809046292, + "per_stream_tps": [ + 205.4, + 216.3, + 212.6, + 208.9 + ], + "ttft_s": [ + 0.02, + 0.08, + 0.06, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021602291148155928, + "decode_tps": 205.41908282904623, + "prefill_tps": 2499.734848940303, + "itl_mean_ms": 4.868096898437715, + "itl_p50_ms": 4.680166952311993, + "itl_p95_ms": 5.101791117340326, + "round_gap_p50_ms": 4.695499781519175, + "round_gap_p95_ms": 5.101791117340326, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08416220918297768, + "decode_tps": 216.32087273706128, + "prefill_tps": 641.6181386422284, + "itl_mean_ms": 4.622762414681561, + "itl_p50_ms": 4.682666156440973, + "itl_p95_ms": 5.108041223138571, + "round_gap_p50_ms": 4.689750261604786, + "round_gap_p95_ms": 5.108041223138571, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06325095798820257, + "decode_tps": 212.56269852196672, + "prefill_tps": 853.7420098850038, + "itl_mean_ms": 4.704494283114578, + "itl_p50_ms": 4.687250126153231, + "itl_p95_ms": 5.111332982778549, + "round_gap_p50_ms": 4.69495914876461, + "round_gap_p95_ms": 5.111332982778549, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.042524250224232674, + "decode_tps": 208.93478903561368, + "prefill_tps": 1269.863659329796, + "itl_mean_ms": 4.786182351994748, + "itl_p50_ms": 4.68333438038826, + "itl_p95_ms": 5.112875252962112, + "round_gap_p50_ms": 4.685292020440102, + "round_gap_p95_ms": 5.112875252962112, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.74538398705715, + "itl_p50_ms": 4.684249870479107, + "itl_p95_ms": 5.108041223138571, + "round_gap_p50_ms": 4.693042021244764, + "round_gap_p95_ms": 5.108082666993141, + "round_gap_mean_ms": 4.840291666798294, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2564.2953356795456 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 820.9407496597628, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2646129997447133, + "end_to_end_tps": 809.7338871312522, + "timing_valid": true, + "steady_tps": 862.4537927590604, + "per_stream_tps": [ + 205.2, + 208.7, + 216.1, + 212.4 + ], + "ttft_s": [ + 0.02, + 0.04, + 0.08, + 0.06 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021629041992127895, + "decode_tps": 205.2433575373338, + "prefill_tps": 2496.643171697287, + "itl_mean_ms": 4.872264866443241, + "itl_p50_ms": 4.6997093595564365, + "itl_p95_ms": 5.112749990075827, + "round_gap_p50_ms": 4.702625330537558, + "round_gap_p95_ms": 5.112749990075827, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04247591691091657, + "decode_tps": 208.74624242419006, + "prefill_tps": 1271.3086362150236, + "itl_mean_ms": 4.790505392513439, + "itl_p50_ms": 4.696625284850597, + "itl_p95_ms": 5.106791853904724, + "round_gap_p50_ms": 4.702416714280844, + "round_gap_p95_ms": 5.106791853904724, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08418737538158894, + "decode_tps": 216.13170296644964, + "prefill_tps": 641.426339225315, + "itl_mean_ms": 4.6268084981277875, + "itl_p50_ms": 4.6928334049880505, + "itl_p95_ms": 5.10375015437603, + "round_gap_p50_ms": 4.7025419771671295, + "round_gap_p95_ms": 5.10375015437603, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06344295758754015, + "decode_tps": 212.3770499431382, + "prefill_tps": 851.158301147759, + "itl_mean_ms": 4.70860669864159, + "itl_p50_ms": 4.699083045125008, + "itl_p95_ms": 5.106625147163868, + "round_gap_p50_ms": 4.705790895968676, + "round_gap_p95_ms": 5.106625147163868, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.749546363931515, + "itl_p50_ms": 4.69766603782773, + "itl_p95_ms": 5.106625147163868, + "round_gap_p50_ms": 4.704874940216541, + "round_gap_p95_ms": 5.106666125357151, + "round_gap_mean_ms": 4.844537291210145, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2561.9407605592132 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + } + ] + } + ] + } + }, + { + "name": "vl-4-2-new.concurrent.json", + "data": { + "label": "vl-4-2-new", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 305.3, + "per_stream_tps": 305.3, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 305.334, + "prefill_tps": 2518.627, + "itl_mean_ms": 3.275, + "itl_p50_ms": 3.321, + "itl_p95_ms": 3.822, + "round_gap_p50_ms": 3.329, + "round_gap_p95_ms": 3.822, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2518.627, + "itl_mean_ms": 3.275, + "itl_p50_ms": 3.321, + "itl_p95_ms": 3.822, + "round_gap_p50_ms": 3.329, + "round_gap_p95_ms": 3.822, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 298.9788004747036, + "makespan_s": 0.8562480001710355, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 305.4331008114392, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8559827501885593, + "end_to_end_tps": 299.0714473435444, + "timing_valid": true, + "per_stream_tps": [ + 305.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021049166098237038, + "decode_tps": 305.4331008114392, + "prefill_tps": 2517.914474742018, + "itl_mean_ms": 3.2740393799601817, + "itl_p50_ms": 3.3194171264767647, + "itl_p95_ms": 3.82199976593256, + "round_gap_p50_ms": 3.328875172883272, + "round_gap_p95_ms": 3.82199976593256, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.2740393799601817, + "itl_p50_ms": 3.3194171264767647, + "itl_p95_ms": 3.82199976593256, + "round_gap_p50_ms": 3.328875172883272, + "round_gap_p95_ms": 3.82199976593256, + "round_gap_mean_ms": 3.352931895139945, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2517.914474742018 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 305.33444744884906, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8562480001710355, + "end_to_end_tps": 298.9788004747036, + "timing_valid": true, + "per_stream_tps": [ + 305.3 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02104320889338851, + "decode_tps": 305.33444744884906, + "prefill_tps": 2518.6272810631976, + "itl_mean_ms": 3.2750972199673747, + "itl_p50_ms": 3.3212918788194656, + "itl_p95_ms": 3.812416922301054, + "round_gap_p50_ms": 3.327874932438135, + "round_gap_p95_ms": 3.812416922301054, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.2750972199673747, + "itl_p50_ms": 3.3212918788194656, + "itl_p95_ms": 3.812416922301054, + "round_gap_p50_ms": 3.327874932438135, + "round_gap_p95_ms": 3.812416922301054, + "round_gap_mean_ms": 3.3540152252677933, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2518.6272810631976 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 304.73870178499806, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8578574578277767, + "end_to_end_tps": 298.4178754454502, + "timing_valid": true, + "per_stream_tps": [ + 304.7 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021027499809861183, + "decode_tps": 304.73870178499806, + "prefill_tps": 2520.5088802399987, + "itl_mean_ms": 3.2814998362286416, + "itl_p50_ms": 3.3265422098338604, + "itl_p95_ms": 3.828125074505806, + "round_gap_p50_ms": 3.341959323734045, + "round_gap_p95_ms": 3.828125074505806, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.2814998362286416, + "itl_p50_ms": 3.3265422098338604, + "itl_p95_ms": 3.828125074505806, + "round_gap_p50_ms": 3.341959323734045, + "round_gap_p95_ms": 3.828125074505806, + "round_gap_mean_ms": 3.3605721214389703, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2520.5088802399987 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 824.3, + "per_stream_tps": 211.4, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.052, + "decode_tps": 211.39, + "prefill_tps": 1055.683, + "itl_mean_ms": 4.731, + "itl_p50_ms": 4.661, + "itl_p95_ms": 5.109, + "round_gap_p50_ms": 4.669, + "round_gap_p95_ms": 5.109, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2552.915, + "itl_mean_ms": 4.732, + "itl_p50_ms": 4.662, + "itl_p95_ms": 5.105, + "round_gap_p50_ms": 4.671, + "round_gap_p95_ms": 5.105, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 813.1783060474971, + "makespan_s": 1.2592564169317484, + "steady_tps": 864.9, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 823.8182544276518, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2595549579709768, + "end_to_end_tps": 812.9855656712007, + "timing_valid": true, + "steady_tps": 864.6641012191415, + "per_stream_tps": [ + 206.0, + 216.8, + 213.1, + 209.5 + ], + "ttft_s": [ + 0.02, + 0.08, + 0.06, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02121445769444108, + "decode_tps": 206.01868776117658, + "prefill_tps": 2498.296245106837, + "itl_mean_ms": 4.853928596803955, + "itl_p50_ms": 4.662207793444395, + "itl_p95_ms": 5.11145917698741, + "round_gap_p50_ms": 4.670875146985054, + "round_gap_p95_ms": 5.11145917698741, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08295950014144182, + "decode_tps": 216.77050685807274, + "prefill_tps": 638.8659515744145, + "itl_mean_ms": 4.613173694587221, + "itl_p50_ms": 4.660832695662975, + "itl_p95_ms": 5.0985002890229225, + "round_gap_p50_ms": 4.674416966736317, + "round_gap_p95_ms": 5.0985002890229225, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06227954104542732, + "decode_tps": 213.10311133802813, + "prefill_tps": 851.001775387864, + "itl_mean_ms": 4.6925640537166124, + "itl_p50_ms": 4.655541852116585, + "itl_p95_ms": 5.115249659866095, + "round_gap_p50_ms": 4.670416936278343, + "round_gap_p95_ms": 5.115249659866095, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.04183833301067352, + "decode_tps": 209.50000365337812, + "prefill_tps": 1266.7808726145706, + "itl_mean_ms": 4.773269606498527, + "itl_p50_ms": 4.667374771088362, + "itl_p95_ms": 5.112333223223686, + "round_gap_p50_ms": 4.677124787122011, + "round_gap_p95_ms": 5.112333223223686, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.733233987901579, + "itl_p50_ms": 4.662207793444395, + "itl_p95_ms": 5.1048328168690205, + "round_gap_p50_ms": 4.674416966736317, + "round_gap_p95_ms": 5.105708260089159, + "round_gap_mean_ms": 4.8278986676596105, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2553.349097974765 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 824.716539293788, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2587307500652969, + "end_to_end_tps": 813.5179028135126, + "timing_valid": true, + "steady_tps": 865.3685166084886, + "per_stream_tps": [ + 213.3, + 206.2, + 217.0, + 209.7 + ], + "ttft_s": [ + 0.06, + 0.02, + 0.08, + 0.04 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.0625753328204155, + "decode_tps": 213.28341661244409, + "prefill_tps": 846.9791147911961, + "itl_mean_ms": 4.68859705964432, + "itl_p50_ms": 4.652500152587891, + "itl_p95_ms": 5.115333944559097, + "round_gap_p50_ms": 4.665791057050228, + "round_gap_p95_ms": 5.115333944559097, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021367833949625492, + "decode_tps": 206.18596299101625, + "prefill_tps": 2480.363715149935, + "itl_mean_ms": 4.849990685561709, + "itl_p50_ms": 4.651082679629326, + "itl_p95_ms": 5.11320773512125, + "round_gap_p50_ms": 4.6613747254014015, + "round_gap_p95_ms": 5.11320773512125, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08304287493228912, + "decode_tps": 217.01063385267983, + "prefill_tps": 638.2245321252997, + "itl_mean_ms": 4.608069117381876, + "itl_p50_ms": 4.652999807149172, + "itl_p95_ms": 5.102749913930893, + "round_gap_p50_ms": 4.660958889871836, + "round_gap_p95_ms": 5.102749913930893, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.042051333002746105, + "decode_tps": 209.67773038401398, + "prefill_tps": 1260.3643265372564, + "itl_mean_ms": 4.769223694707833, + "itl_p50_ms": 4.658292047679424, + "itl_p95_ms": 5.104959011077881, + "round_gap_p50_ms": 4.666458815336227, + "round_gap_p95_ms": 5.104959011077881, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.728970139323935, + "itl_p50_ms": 4.652500152587891, + "itl_p95_ms": 5.104667041450739, + "round_gap_p50_ms": 4.663750063627958, + "round_gap_p95_ms": 5.104959011077881, + "round_gap_mean_ms": 4.823549542110413, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2548.8287658636923 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 824.30315020995, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2592564169317484, + "end_to_end_tps": 813.1783060474971, + "timing_valid": true, + "steady_tps": 864.8652765899135, + "per_stream_tps": [ + 209.6, + 216.9, + 213.2, + 206.1 + ], + "ttft_s": [ + 0.04, + 0.08, + 0.06, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.041916958056390285, + "decode_tps": 209.56784030886928, + "prefill_tps": 1264.4047291957554, + "itl_mean_ms": 4.771724509477031, + "itl_p50_ms": 4.661541897803545, + "itl_p95_ms": 5.109417252242565, + "round_gap_p50_ms": 4.6793329529464245, + "round_gap_p95_ms": 5.109417252242565, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08296679193153977, + "decode_tps": 216.88572299343156, + "prefill_tps": 638.8098028875585, + "itl_mean_ms": 4.610723039756219, + "itl_p50_ms": 4.664457868784666, + "itl_p95_ms": 5.098292138427496, + "round_gap_p50_ms": 4.666541237384081, + "round_gap_p95_ms": 5.098292138427496, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06234437506645918, + "decode_tps": 213.15960326997163, + "prefill_tps": 850.1167899028892, + "itl_mean_ms": 4.691320422160275, + "itl_p50_ms": 4.660500213503838, + "itl_p95_ms": 5.108000244945288, + "round_gap_p50_ms": 4.67341672629118, + "round_gap_p95_ms": 5.108000244945288, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021367874927818775, + "decode_tps": 206.0814500485534, + "prefill_tps": 2480.358958438092, + "itl_mean_ms": 4.852450328568618, + "itl_p50_ms": 4.662625025957823, + "itl_p95_ms": 5.100875161588192, + "round_gap_p50_ms": 4.666999913752079, + "round_gap_p95_ms": 5.100875161588192, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.731554574990535, + "itl_p50_ms": 4.662625025957823, + "itl_p95_ms": 5.102917086333036, + "round_gap_p50_ms": 4.670874681323767, + "round_gap_p95_ms": 5.1052081398665905, + "round_gap_mean_ms": 4.826185666490346, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2552.9147630184307 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 303.2, + "per_stream_tps": 303.2, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 303.226, + "prefill_tps": 2536.035, + "itl_mean_ms": 3.298, + "itl_p50_ms": 3.364, + "itl_p95_ms": 3.839, + "round_gap_p50_ms": 3.38, + "round_gap_p95_ms": 3.839, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2536.035, + "itl_mean_ms": 3.298, + "itl_p50_ms": 3.364, + "itl_p95_ms": 3.839, + "round_gap_p50_ms": 3.38, + "round_gap_p95_ms": 3.839, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 296.88552854806846, + "makespan_s": 0.8622852088883519, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 303.22632807884486, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8622852088883519, + "end_to_end_tps": 296.88552854806846, + "timing_valid": true, + "per_stream_tps": [ + 303.2 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021275208797305822, + "decode_tps": 303.22632807884486, + "prefill_tps": 2538.1654541899616, + "itl_mean_ms": 3.297866667237352, + "itl_p50_ms": 3.364040981978178, + "itl_p95_ms": 3.827792126685381, + "round_gap_p50_ms": 3.379624802619219, + "round_gap_p95_ms": 3.827792126685381, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.297866667237352, + "itl_p50_ms": 3.364040981978178, + "itl_p95_ms": 3.827792126685381, + "round_gap_p50_ms": 3.379624802619219, + "round_gap_p95_ms": 3.827792126685381, + "round_gap_mean_ms": 3.37733333391777, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2538.1654541899616 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 302.3949831566762, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8646222911775112, + "end_to_end_tps": 296.08304413636955, + "timing_valid": true, + "per_stream_tps": [ + 302.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021304207853972912, + "decode_tps": 302.3949831566762, + "prefill_tps": 2534.7105309024582, + "itl_mean_ms": 3.3069331691983868, + "itl_p50_ms": 3.3746669068932533, + "itl_p95_ms": 3.8391659036278725, + "round_gap_p50_ms": 3.3855829387903214, + "round_gap_p95_ms": 3.8391659036278725, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.3069331691983868, + "itl_p50_ms": 3.3746669068932533, + "itl_p95_ms": 3.8391659036278725, + "round_gap_p50_ms": 3.3855829387903214, + "round_gap_p95_ms": 3.8391659036278725, + "round_gap_mean_ms": 3.3866183058055768, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2534.7105309024582 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 303.70215892973494, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8610290409997106, + "end_to_end_tps": 297.3186591973337, + "timing_valid": true, + "per_stream_tps": [ + 303.7 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02129308320581913, + "decode_tps": 303.70215892973494, + "prefill_tps": 2536.0347995654515, + "itl_mean_ms": 3.2926996749844037, + "itl_p50_ms": 3.328540828078985, + "itl_p95_ms": 3.838915843516588, + "round_gap_p50_ms": 3.340708091855049, + "round_gap_p95_ms": 3.838915843516588, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.2926996749844037, + "itl_p50_ms": 3.328540828078985, + "itl_p95_ms": 3.838915843516588, + "round_gap_p50_ms": 3.340708091855049, + "round_gap_p95_ms": 3.838915843516588, + "round_gap_mean_ms": 3.3720418358274014, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2536.0347995654515 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 819.8, + "per_stream_tps": 210.3, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.053, + "decode_tps": 210.37, + "prefill_tps": 1062.711, + "itl_mean_ms": 4.754, + "itl_p50_ms": 4.708, + "itl_p95_ms": 5.115, + "round_gap_p50_ms": 4.712, + "round_gap_p95_ms": 5.115, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2564.148, + "itl_mean_ms": 4.757, + "itl_p50_ms": 4.708, + "itl_p95_ms": 5.117, + "round_gap_p50_ms": 4.715, + "round_gap_p95_ms": 5.117, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 808.5299376786163, + "makespan_s": 1.266496084164828, + "steady_tps": 861.2, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 819.7636992008628, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.266496084164828, + "end_to_end_tps": 808.5299376786163, + "timing_valid": true, + "steady_tps": 861.1517343503975, + "per_stream_tps": [ + 212.1, + 208.4, + 204.9, + 215.8 + ], + "ttft_s": [ + 0.06, + 0.04, + 0.02, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.06347250007092953, + "decode_tps": 212.065163416054, + "prefill_tps": 850.7621401340083, + "itl_mean_ms": 4.715531697387204, + "itl_p50_ms": 4.707875195890665, + "itl_p95_ms": 5.118833854794502, + "round_gap_p50_ms": 4.712875001132488, + "round_gap_p95_ms": 5.118833854794502, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.042497833259403706, + "decode_tps": 208.44745581421003, + "prefill_tps": 1270.6530158934904, + "itl_mean_ms": 4.797372057595673, + "itl_p50_ms": 4.710042383521795, + "itl_p95_ms": 5.114249885082245, + "round_gap_p50_ms": 4.714958369731903, + "round_gap_p95_ms": 5.114249885082245, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021629500202834606, + "decode_tps": 204.9409248002157, + "prefill_tps": 2496.5902814954156, + "itl_mean_ms": 4.879454901332364, + "itl_p50_ms": 4.698332864791155, + "itl_p95_ms": 5.121542140841484, + "round_gap_p50_ms": 4.711624700576067, + "round_gap_p95_ms": 5.121542140841484, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08439383376389742, + "decode_tps": 215.81228398375177, + "prefill_tps": 639.8571742939411, + "itl_mean_ms": 4.633656534932408, + "itl_p50_ms": 4.70854202285409, + "itl_p95_ms": 5.106708034873009, + "round_gap_p50_ms": 4.7113336622715, + "round_gap_p95_ms": 5.106708034873009, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.756503797811913, + "itl_p50_ms": 4.70783282071352, + "itl_p95_ms": 5.116833373904228, + "round_gap_p50_ms": 4.714958369731903, + "round_gap_p95_ms": 5.116958171129227, + "round_gap_mean_ms": 4.851633873768151, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2559.4286971757642 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 820.8469918070812, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2646354171447456, + "end_to_end_tps": 809.7195334857498, + "timing_valid": true, + "steady_tps": 862.3199445289711, + "per_stream_tps": [ + 208.7, + 205.2, + 212.3, + 216.1 + ], + "ttft_s": [ + 0.04, + 0.02, + 0.06, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.04239924997091293, + "decode_tps": 208.72257070382486, + "prefill_tps": 1273.6074349674936, + "itl_mean_ms": 4.791048695059383, + "itl_p50_ms": 4.700874909758568, + "itl_p95_ms": 5.125041119754314, + "round_gap_p50_ms": 4.705834202468395, + "round_gap_p95_ms": 5.125041119754314, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.02147645829245448, + "decode_tps": 205.21717049150266, + "prefill_tps": 2514.3810615632237, + "itl_mean_ms": 4.872886599132828, + "itl_p50_ms": 4.694417119026184, + "itl_p95_ms": 5.1190839149057865, + "round_gap_p50_ms": 4.703500308096409, + "round_gap_p95_ms": 5.1190839149057865, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06317495787516236, + "decode_tps": 212.34553327745843, + "prefill_tps": 854.7690701544645, + "itl_mean_ms": 4.709305557623213, + "itl_p50_ms": 4.707582760602236, + "itl_p95_ms": 5.119000095874071, + "round_gap_p50_ms": 4.710166715085506, + "round_gap_p95_ms": 5.119000095874071, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.0841696672141552, + "decode_tps": 216.10510596083256, + "prefill_tps": 641.5612867115931, + "itl_mean_ms": 4.627377939793994, + "itl_p50_ms": 4.683542065322399, + "itl_p95_ms": 5.113916005939245, + "round_gap_p50_ms": 4.7006248496472836, + "round_gap_p95_ms": 5.113916005939245, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.750154697902355, + "itl_p50_ms": 4.69700014218688, + "itl_p95_ms": 5.117999855428934, + "round_gap_p50_ms": 4.708749707788229, + "round_gap_p95_ms": 5.118417087942362, + "round_gap_mean_ms": 4.845157791860402, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2566.2451468463723 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 819.4874381361259, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2666507917456329, + "end_to_end_tps": 808.4311845641181, + "timing_valid": true, + "steady_tps": 860.9560294752682, + "per_stream_tps": [ + 212.0, + 208.4, + 215.7, + 204.9 + ], + "ttft_s": [ + 0.06, + 0.04, + 0.08, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.0633419998921454, + "decode_tps": 212.0175570477486, + "prefill_tps": 852.5149204626892, + "itl_mean_ms": 4.71659052167453, + "itl_p50_ms": 4.711291752755642, + "itl_p95_ms": 5.105957854539156, + "round_gap_p50_ms": 4.714875016361475, + "round_gap_p95_ms": 5.105957854539156, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04243100015446544, + "decode_tps": 208.40177907319747, + "prefill_tps": 1272.6544225547095, + "itl_mean_ms": 4.798423528086905, + "itl_p50_ms": 4.714708775281906, + "itl_p95_ms": 5.114249885082245, + "round_gap_p50_ms": 4.716749768704176, + "round_gap_p95_ms": 5.114249885082245, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08410716615617275, + "decode_tps": 215.71975211270322, + "prefill_tps": 642.0380387056574, + "itl_mean_ms": 4.635644117918084, + "itl_p50_ms": 4.705499857664108, + "itl_p95_ms": 5.104249808937311, + "round_gap_p50_ms": 4.714500159025192, + "round_gap_p95_ms": 5.104249808937311, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021647291723638773, + "decode_tps": 204.91303784618282, + "prefill_tps": 2494.5383787216288, + "itl_mean_ms": 4.880118954415415, + "itl_p50_ms": 4.714583978056908, + "itl_p95_ms": 5.115125328302383, + "round_gap_p50_ms": 4.71804104745388, + "round_gap_p95_ms": 5.115125328302383, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.757694280523734, + "itl_p50_ms": 4.712875001132488, + "itl_p95_ms": 5.10937487706542, + "round_gap_p50_ms": 4.7153751365840435, + "round_gap_p95_ms": 5.1093753427267075, + "round_gap_mean_ms": 4.852848166134208, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2564.1482260103426 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + } + ] + } + ] + } + }, + { + "name": "vl-4-3-old.concurrent.json", + "data": { + "label": "vl-4-3-old", + "tokens": 256, + "mixed": false, + "cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 301.5, + "per_stream_tps": 301.5, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 301.493, + "prefill_tps": 2517.67, + "itl_mean_ms": 3.317, + "itl_p50_ms": 3.356, + "itl_p95_ms": 3.87, + "round_gap_p50_ms": 3.383, + "round_gap_p95_ms": 3.87, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2517.67, + "itl_mean_ms": 3.317, + "itl_p50_ms": 3.356, + "itl_p95_ms": 3.87, + "round_gap_p50_ms": 3.383, + "round_gap_p95_ms": 3.87, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 295.3067848496112, + "makespan_s": 0.8668950838036835, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 300.74070679920675, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8690017908811569, + "end_to_end_tps": 294.5908773564427, + "timing_valid": true, + "per_stream_tps": [ + 300.7 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021039416082203388, + "decode_tps": 300.74070679920675, + "prefill_tps": 2519.081318270573, + "itl_mean_ms": 3.3251235279820714, + "itl_p50_ms": 3.3767083659768105, + "itl_p95_ms": 3.889415878802538, + "round_gap_p50_ms": 3.383458126336336, + "round_gap_p95_ms": 3.889415878802538, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.3251235279820714, + "itl_p50_ms": 3.3767083659768105, + "itl_p95_ms": 3.889415878802538, + "round_gap_p50_ms": 3.383458126336336, + "round_gap_p95_ms": 3.889415878802538, + "round_gap_mean_ms": 3.4052469864876636, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2519.081318270573 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 301.7383233241175, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8662314577959478, + "end_to_end_tps": 295.53302145291536, + "timing_valid": true, + "per_stream_tps": [ + 301.7 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021075000055134296, + "decode_tps": 301.7383233241175, + "prefill_tps": 2514.8279886760015, + "itl_mean_ms": 3.3141299023055564, + "itl_p50_ms": 3.353457897901535, + "itl_p95_ms": 3.870166838169098, + "round_gap_p50_ms": 3.383500035852194, + "round_gap_p95_ms": 3.870166838169098, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.3141299023055564, + "itl_p50_ms": 3.353457897901535, + "itl_p95_ms": 3.870166838169098, + "round_gap_p50_ms": 3.383500035852194, + "round_gap_p95_ms": 3.870166838169098, + "round_gap_mean_ms": 3.393988454168341, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2514.8279886760015 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 301.4927738013548, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8668950838036835, + "end_to_end_tps": 295.3067848496112, + "timing_valid": true, + "per_stream_tps": [ + 301.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.021051208954304457, + "decode_tps": 301.4927738013548, + "prefill_tps": 2517.670130729608, + "itl_mean_ms": 3.3168290814786565, + "itl_p50_ms": 3.355707973241806, + "itl_p95_ms": 3.858125302940607, + "round_gap_p50_ms": 3.381749615073204, + "round_gap_p95_ms": 3.858125302940607, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.3168290814786565, + "itl_p50_ms": 3.355707973241806, + "itl_p95_ms": 3.858125302940607, + "round_gap_p50_ms": 3.381749615073204, + "round_gap_p95_ms": 3.858125302940607, + "round_gap_mean_ms": 3.3967526738034435, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2517.670130729608 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 813.1, + "per_stream_tps": 208.4, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.052, + "decode_tps": 208.421, + "prefill_tps": 1054.462, + "itl_mean_ms": 4.798, + "itl_p50_ms": 4.73, + "itl_p95_ms": 5.176, + "round_gap_p50_ms": 4.741, + "round_gap_p95_ms": 5.176, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2544.679, + "itl_mean_ms": 4.796, + "itl_p50_ms": 4.732, + "itl_p95_ms": 5.174, + "round_gap_p50_ms": 4.742, + "round_gap_p95_ms": 5.175, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 802.6582045812256, + "makespan_s": 1.2757609579712152, + "steady_tps": 853.1, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 812.1437254069934, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.277591582853347, + "end_to_end_tps": 801.5080983181019, + "timing_valid": true, + "steady_tps": 851.8179166829875, + "per_stream_tps": [ + 206.5, + 213.6, + 210.0, + 203.1 + ], + "ttft_s": [ + 0.04, + 0.08, + 0.06, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.04198608314618468, + "decode_tps": 206.474841855525, + "prefill_tps": 1262.3230372661271, + "itl_mean_ms": 4.843205065630815, + "itl_p50_ms": 4.740999545902014, + "itl_p95_ms": 5.186083260923624, + "round_gap_p50_ms": 4.747250117361546, + "round_gap_p95_ms": 5.186083260923624, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.083232541102916, + "decode_tps": 213.56754856768956, + "prefill_tps": 636.7701778378502, + "itl_mean_ms": 4.682359313044478, + "itl_p50_ms": 4.732457920908928, + "itl_p95_ms": 5.178249906748533, + "round_gap_p50_ms": 4.743667319417, + "round_gap_p95_ms": 5.178249906748533, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06256154086440802, + "decode_tps": 209.98363052184789, + "prefill_tps": 847.1658349155578, + "itl_mean_ms": 4.762275980822012, + "itl_p50_ms": 4.734000191092491, + "itl_p95_ms": 5.181207787245512, + "round_gap_p50_ms": 4.744041245430708, + "round_gap_p95_ms": 5.181207787245512, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021368875168263912, + "decode_tps": 203.08330228547408, + "prefill_tps": 2480.2428570837087, + "itl_mean_ms": 4.924087745009684, + "itl_p50_ms": 4.73591685295105, + "itl_p95_ms": 5.185749847441912, + "round_gap_p50_ms": 4.7504170797765255, + "round_gap_p95_ms": 5.185749847441912, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.802982026126747, + "itl_p50_ms": 4.734000191092491, + "itl_p95_ms": 5.180000327527523, + "round_gap_p50_ms": 4.747250117361546, + "round_gap_p95_ms": 5.180208012461662, + "round_gap_mean_ms": 4.899041666649282, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2544.9390902674863 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 813.7550937998988, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.275354083161801, + "end_to_end_tps": 802.9142757447758, + "timing_valid": true, + "steady_tps": 853.4371498679028, + "per_stream_tps": [ + 206.9, + 203.5, + 214.0, + 210.4 + ], + "ttft_s": [ + 0.04, + 0.02, + 0.08, + 0.06 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.0420315838418901, + "decode_tps": 206.85883041659145, + "prefill_tps": 1260.9565273430978, + "itl_mean_ms": 4.834214705681684, + "itl_p50_ms": 4.714292008429766, + "itl_p95_ms": 5.176832899451256, + "round_gap_p50_ms": 4.726292099803686, + "round_gap_p95_ms": 5.176832899451256, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.021371833980083466, + "decode_tps": 203.45660810311784, + "prefill_tps": 2479.899481223324, + "itl_mean_ms": 4.915052940886395, + "itl_p50_ms": 4.709708970040083, + "itl_p95_ms": 5.171915981918573, + "round_gap_p50_ms": 4.717499949038029, + "round_gap_p95_ms": 5.171915981918573, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08321108296513557, + "decode_tps": 214.00629955086413, + "prefill_tps": 636.9343855578271, + "itl_mean_ms": 4.672759643518457, + "itl_p50_ms": 4.708041902631521, + "itl_p95_ms": 5.174540914595127, + "round_gap_p50_ms": 4.719082731753588, + "round_gap_p95_ms": 5.174540914595127, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06271466612815857, + "decode_tps": 210.37865994549497, + "prefill_tps": 845.0973794820741, + "itl_mean_ms": 4.753333823207547, + "itl_p50_ms": 4.7188750468194485, + "itl_p95_ms": 5.1766252145171165, + "round_gap_p50_ms": 4.719958174973726, + "round_gap_p95_ms": 5.1766252145171165, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.793840278323521, + "itl_p50_ms": 4.713582806289196, + "itl_p95_ms": 5.1742917858064175, + "round_gap_p50_ms": 4.7211661003530025, + "round_gap_p95_ms": 5.174540914595127, + "round_gap_mean_ms": 4.889717083889991, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2543.741821565 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 813.1482616145435, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2757609579712152, + "end_to_end_tps": 802.6582045812256, + "timing_valid": true, + "steady_tps": 853.1122511205176, + "per_stream_tps": [ + 203.4, + 206.8, + 210.3, + 213.8 + ], + "ttft_s": [ + 0.02, + 0.04, + 0.06, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 53, + "ttft_s": 0.02130983304232359, + "decode_tps": 203.37174401647204, + "prefill_tps": 2487.1147462646177, + "itl_mean_ms": 4.917103921373685, + "itl_p50_ms": 4.732375033199787, + "itl_p95_ms": 5.175791680812836, + "round_gap_p50_ms": 4.737125243991613, + "round_gap_p95_ms": 5.175791680812836, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.041928709018975496, + "decode_tps": 206.77874808055807, + "prefill_tps": 1264.0503664449582, + "itl_mean_ms": 4.836086925192206, + "itl_p50_ms": 4.72754193469882, + "itl_p95_ms": 5.1739998161792755, + "round_gap_p50_ms": 4.742375109344721, + "round_gap_p95_ms": 5.1739998161792755, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.06250233296304941, + "decode_tps": 210.2917973654212, + "prefill_tps": 847.9683475388499, + "itl_mean_ms": 4.75529722285037, + "itl_p50_ms": 4.740624688565731, + "itl_p95_ms": 5.173124838620424, + "round_gap_p50_ms": 4.742166958749294, + "round_gap_p95_ms": 5.173124838620424, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 53, + "ttft_s": 0.08331108326092362, + "decode_tps": 213.84909071299077, + "prefill_tps": 636.1698579048391, + "itl_mean_ms": 4.676194772051246, + "itl_p50_ms": 4.719000309705734, + "itl_p95_ms": 5.170250311493874, + "round_gap_p50_ms": 4.740416072309017, + "round_gap_p95_ms": 5.170250311493874, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.7961707103668765, + "itl_p50_ms": 4.732124973088503, + "itl_p95_ms": 5.171541590243578, + "round_gap_p50_ms": 4.7417497262358665, + "round_gap_p95_ms": 5.173082929104567, + "round_gap_mean_ms": 4.892094124574214, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2544.6794316193564 + }, + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 299.7, + "per_stream_tps": 299.7, + "ttft_s_max": 0.02, + "request": { + "ttft_s": 0.021, + "decode_tps": 299.669, + "prefill_tps": 2528.381, + "itl_mean_ms": 3.337, + "itl_p50_ms": 3.403, + "itl_p95_ms": 3.871, + "round_gap_p50_ms": 3.415, + "round_gap_p95_ms": 3.871, + "tokens_per_chunk": 1.024 + }, + "pooled": { + "aggregate_prefill_tps": 2528.381, + "itl_mean_ms": 3.337, + "itl_p50_ms": 3.403, + "itl_p95_ms": 3.871, + "round_gap_p50_ms": 3.415, + "round_gap_p95_ms": 3.871, + "tokens_per_chunk": 1.024 + }, + "timing_valid": true, + "end_to_end_tps": 293.45827311672844, + "makespan_s": 0.8723557093180716, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 299.6691740076389, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8723557093180716, + "end_to_end_tps": 293.45827311672844, + "timing_valid": true, + "per_stream_tps": [ + 299.7 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.02136095892637968, + "decode_tps": 299.6691740076389, + "prefill_tps": 2527.976397787685, + "itl_mean_ms": 3.3370132357174276, + "itl_p50_ms": 3.4065409563481808, + "itl_p95_ms": 3.8730003871023655, + "round_gap_p50_ms": 3.4261667169630527, + "round_gap_p95_ms": 3.8730003871023655, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.3370132357174276, + "itl_p50_ms": 3.4065409563481808, + "itl_p95_ms": 3.8730003871023655, + "round_gap_p50_ms": 3.4261667169630527, + "round_gap_p95_ms": 3.8730003871023655, + "round_gap_mean_ms": 3.4174231932045944, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2527.976397787685 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 299.6263042947447, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8724674172699451, + "end_to_end_tps": 293.420699653237, + "timing_valid": true, + "per_stream_tps": [ + 299.6 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021352792158722878, + "decode_tps": 299.6263042947447, + "prefill_tps": 2528.943268805262, + "itl_mean_ms": 3.337490686452856, + "itl_p50_ms": 3.403250128030777, + "itl_p95_ms": 3.8609160110354424, + "round_gap_p50_ms": 3.4145419485867023, + "round_gap_p95_ms": 3.8609160110354424, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.337490686452856, + "itl_p50_ms": 3.403250128030777, + "itl_p95_ms": 3.8609160110354424, + "round_gap_p50_ms": 3.4145419485867023, + "round_gap_p95_ms": 3.8609160110354424, + "round_gap_mean_ms": 3.4179121487770217, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2528.943268805262 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 299.7731569790372, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8720547081902623, + "end_to_end_tps": 293.5595640911862, + "timing_valid": true, + "per_stream_tps": [ + 299.8 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021357541903853416, + "decode_tps": 299.7731569790372, + "prefill_tps": 2528.3808522111385, + "itl_mean_ms": 3.3358557186290327, + "itl_p50_ms": 3.391875419765711, + "itl_p95_ms": 3.8707908242940903, + "round_gap_p50_ms": 3.402499947696924, + "round_gap_p95_ms": 3.8707908242940903, + "tokens_per_chunk": 1.024 + } + ], + "pooled": { + "streams_timed": 1, + "itl_mean_ms": 3.3358557186290327, + "itl_p50_ms": 3.391875419765711, + "itl_p95_ms": 3.8707908242940903, + "round_gap_p50_ms": 3.402499947696924, + "round_gap_p95_ms": 3.8707908242940903, + "round_gap_mean_ms": 3.416237784138166, + "tokens_per_chunk": 1.024, + "aggregate_prefill_tps": 2528.3808522111385 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 809.5, + "per_stream_tps": 207.4, + "ttft_s_max": 0.08, + "request": { + "ttft_s": 0.053, + "decode_tps": 207.436, + "prefill_tps": 1060.277, + "itl_mean_ms": 4.821, + "itl_p50_ms": 4.769, + "itl_p95_ms": 5.182, + "round_gap_p50_ms": 4.777, + "round_gap_p95_ms": 5.182, + "tokens_per_chunk": 1.02 + }, + "pooled": { + "aggregate_prefill_tps": 2559.215, + "itl_mean_ms": 4.818, + "itl_p50_ms": 4.767, + "itl_p95_ms": 5.178, + "round_gap_p50_ms": 4.779, + "round_gap_p95_ms": 5.18, + "tokens_per_chunk": 1.02 + }, + "timing_valid": true, + "end_to_end_tps": 798.6716623047198, + "makespan_s": 1.282128875143826, + "steady_tps": 849.9, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 808.4141918290359, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.283813082613051, + "end_to_end_tps": 797.6239016942935, + "timing_valid": true, + "steady_tps": 848.699556565283, + "per_stream_tps": [ + 209.0, + 205.5, + 212.7, + 202.1 + ], + "ttft_s": [ + 0.06, + 0.04, + 0.08, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.06342749996110797, + "decode_tps": 209.04194539839727, + "prefill_tps": 851.3657330513001, + "itl_mean_ms": 4.783728921457248, + "itl_p50_ms": 4.786042030900717, + "itl_p95_ms": 5.183708854019642, + "round_gap_p50_ms": 4.789041820913553, + "round_gap_p95_ms": 5.183708854019642, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04245304176583886, + "decode_tps": 205.5143185847533, + "prefill_tps": 1271.9936606156864, + "itl_mean_ms": 4.865841012375028, + "itl_p50_ms": 4.777665715664625, + "itl_p95_ms": 5.191542208194733, + "round_gap_p50_ms": 4.785709083080292, + "round_gap_p95_ms": 5.191542208194733, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08427533321082592, + "decode_tps": 212.68765256720621, + "prefill_tps": 640.7568851126562, + "itl_mean_ms": 4.701730391631524, + "itl_p50_ms": 4.781499970704317, + "itl_p95_ms": 5.187375005334616, + "round_gap_p50_ms": 4.788499791175127, + "round_gap_p95_ms": 5.187375005334616, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021611582953482866, + "decode_tps": 202.10909436262233, + "prefill_tps": 2498.6600989030053, + "itl_mean_ms": 4.94782287335279, + "itl_p50_ms": 4.773666616529226, + "itl_p95_ms": 5.1865833811461926, + "round_gap_p50_ms": 4.784833174198866, + "round_gap_p95_ms": 5.1865833811461926, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.824780799704147, + "itl_p50_ms": 4.777665715664625, + "itl_p95_ms": 5.185333080589771, + "round_gap_p50_ms": 4.788625054061413, + "round_gap_p95_ms": 5.186291877180338, + "round_gap_mean_ms": 4.92127641569823, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2559.2480445832725 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 809.6224420213014, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.281973000150174, + "end_to_end_tps": 798.7687727276984, + "timing_valid": true, + "steady_tps": 850.0270139150657, + "per_stream_tps": [ + 202.4, + 205.8, + 209.4, + 213.0 + ], + "ttft_s": [ + 0.02, + 0.04, + 0.06, + 0.08 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.021594665944576263, + "decode_tps": 202.4140319173744, + "prefill_tps": 2500.6175200206185, + "itl_mean_ms": 4.940368958255824, + "itl_p50_ms": 4.767375066876411, + "itl_p95_ms": 5.182249937206507, + "round_gap_p50_ms": 4.770833067595959, + "round_gap_p95_ms": 5.182249937206507, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.04250462492927909, + "decode_tps": 205.83091407224964, + "prefill_tps": 1270.4499825571306, + "itl_mean_ms": 4.858356697813553, + "itl_p50_ms": 4.764708224684, + "itl_p95_ms": 5.179708357900381, + "round_gap_p50_ms": 4.7722081653773785, + "round_gap_p95_ms": 5.179708357900381, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06336120795458555, + "decode_tps": 209.36423251897628, + "prefill_tps": 852.2564790542623, + "itl_mean_ms": 4.776365036035285, + "itl_p50_ms": 4.764582961797714, + "itl_p95_ms": 5.190333351492882, + "round_gap_p50_ms": 4.774207714945078, + "round_gap_p95_ms": 5.190333351492882, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.0844008750282228, + "decode_tps": 213.01714348300763, + "prefill_tps": 639.8037932893817, + "itl_mean_ms": 4.694457843388412, + "itl_p50_ms": 4.752084147185087, + "itl_p95_ms": 5.174166988581419, + "round_gap_p50_ms": 4.775624722242355, + "round_gap_p95_ms": 5.174166988581419, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.817387133873269, + "itl_p50_ms": 4.764708224684, + "itl_p95_ms": 5.178249906748533, + "round_gap_p50_ms": 4.774207714945078, + "round_gap_p95_ms": 5.179708357900381, + "round_gap_mean_ms": 4.913734876550734, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2559.215173157527 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 809.5194997364449, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.282128875143826, + "end_to_end_tps": 798.6716623047198, + "timing_valid": true, + "steady_tps": 849.8960489862131, + "per_stream_tps": [ + 205.8, + 209.3, + 213.0, + 202.4 + ], + "ttft_s": [ + 0.04, + 0.06, + 0.08, + 0.02 + ], + "errors": [], + "requests": [ + { + "prompt_tokens": 54, + "ttft_s": 0.04257674980908632, + "decode_tps": 205.8061063472422, + "prefill_tps": 1268.2978442961337, + "itl_mean_ms": 4.858942320752962, + "itl_p50_ms": 4.77870786562562, + "itl_p95_ms": 5.1724170334637165, + "round_gap_p50_ms": 4.7815409488976, + "round_gap_p95_ms": 5.1724170334637165, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.06344099994748831, + "decode_tps": 209.33530084217483, + "prefill_tps": 851.1845658910979, + "itl_mean_ms": 4.777025164780664, + "itl_p50_ms": 4.770334344357252, + "itl_p95_ms": 5.181749816983938, + "round_gap_p50_ms": 4.774667322635651, + "round_gap_p95_ms": 5.181749816983938, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.08427166705951095, + "decode_tps": 212.98778645835657, + "prefill_tps": 640.7847605751799, + "itl_mean_ms": 4.695104900747538, + "itl_p50_ms": 4.757208749651909, + "itl_p95_ms": 5.17683383077383, + "round_gap_p50_ms": 4.769249819219112, + "round_gap_p95_ms": 5.17683383077383, + "tokens_per_chunk": 1.0199203187250996 + }, + { + "prompt_tokens": 54, + "ttft_s": 0.021661541890352964, + "decode_tps": 202.3851687411955, + "prefill_tps": 2492.897332670906, + "itl_mean_ms": 4.9410735293492385, + "itl_p50_ms": 4.765375051647425, + "itl_p95_ms": 5.177541635930538, + "round_gap_p50_ms": 4.7790417447686195, + "round_gap_p95_ms": 5.177541635930538, + "tokens_per_chunk": 1.0199203187250996 + } + ], + "pooled": { + "streams_timed": 4, + "itl_mean_ms": 4.8180364789076, + "itl_p50_ms": 4.766667261719704, + "itl_p95_ms": 5.177249666303396, + "round_gap_p50_ms": 4.7790417447686195, + "round_gap_p95_ms": 5.177541635930538, + "round_gap_mean_ms": 4.914397208485752, + "tokens_per_chunk": 1.0199203187250996, + "aggregate_prefill_tps": 2558.741469392942 + }, + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ], + "comparisons": [ + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + }, + { + "concurrent": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "alone": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + }, + "serial": { + "tokens": 256, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a", + "complete": true, + "error": null + } + } + ] + } + ] + } + ] + } + } + ] + } +} diff --git a/docs/receipts/lfm3/local-standard-phase1.json b/docs/receipts/lfm3/local-standard-phase1.json new file mode 100644 index 0000000000..a76fee8f10 --- /dev/null +++ b/docs/receipts/lfm3/local-standard-phase1.json @@ -0,0 +1,11049 @@ +{ + "schema_version": 1, + "scope": "Actual locked local Apple M4 Pro phase1: text4 and VL4, code/prose text requests, exact256-token solo/four-stream decode, cold2048/8192 with16 outputs. No performance gate pass.", + "status": { + "actual_exit_code": 0, + "server_legs": 12, + "measured_decode_requests": 360, + "cold_requests": 24, + "all_lengths_validated": true, + "performance_gate_claim": false + }, + "host": { + "os": "ProductName:\t\tmacOS\nProductVersion:\t\t26.5.1\nBuildVersion:\t\t25F80\n", + "gpu": "Apple M4 Pro", + "gpu_cores": 20, + "physical_memory_bytes": 51539607552 + }, + "provenance": { + "source_base": "21728fcd8498c809dc2737e47f4e8b6977d9f5a1", + "native_artifacts": { + "public": { + "path": "$PACKET/artifacts/tensorfold-native-public", + "sha256": "ad3ca5a0d39daa399b5bde058514291ec52f87a50799528127c654d30a139672" + }, + "combined": { + "path": "$PACKET/artifacts/tensorfold-native-combined", + "sha256": "ad72d7ec17b681ede54676ba75f7d4d1b39282aa0dae7cf27d8b201660fce489" + } + }, + "standard": { + "python": "$PACKET/standard-venv/bin/python", + "module": "mlx_vlm.server", + "source_commit": "230fc308efb577d87446306bfb8a3e9f1a129558", + "mlx": "0.32.4", + "mlx_metal": "0.32.4", + "mlx_vlm": "0.7.11", + "dependency_provenance": "Core source/MLX pins match prior standard procedure. Other dependencies are newly resolved and fully frozen; not asserted identical to the old M5 environment.", + "memory_policy": "Unmodified MLX defaults; no memory/wired limit overrides.", + "request_context_limit": 70000, + "context_policy": "Pinned --max-kv-size70000 sets request budget; LFM3 family cache construction remains lazy and is not claimed allocation-identical to native.", + "checkpoint_context_caps": { + "text": 262144, + "vl": 128000 + } + }, + "versions": { + "python": "3.13.15 (main, Aug 5 2026, 12:25:43) [Clang 21.0.0 (clang-2100.1.1.101)]", + "packages": { + "shellingham": "1.5.4", + "uvicorn": "0.54.0", + "xxhash": "4.0.2", + "filelock": "4.2.4", + "mlx": "0.32.4", + "starlette": "1.10.0", + "huggingface_hub": "2.3.0", + "click": "8.5.0", + "packaging": "26.3", + "tokenizers": "0.23.3", + "Pygments": "2.21.0", + "pyarrow": "26.0.0", + "sounddevice": "0.5.6", + "requests": "2.34.2", + "rich": "15.0.0", + "certifi": "2026.10.23", + "yarl": "1.25.1", + "websockets": "17.3", + "mlx-vlm": "0.7.11", + "opentelemetry-api": "1.45.1", + "pandas": "3.1.0", + "datasets": "5.1.0", + "sentencepiece": "0.2.2", + "propcache": "0.5.4", + "llguidance": "1.9.1", + "numpy": "2.5.4", + "aiohappyeyeballs": "2.7.1", + "cryptography": "50.0.2", + "pycparser": "3.12", + "pillow": "13.0.0", + "fsspec": "2026.7.0", + "hf-xet": "1.7.0", + "safetensors": "0.8.0", + "miniaudio": "1.71", + "tqdm": "4.70.1", + "urllib3": "2.8.0", + "h11": "0.16.0", + "pydantic_core": "2.50.1", + "mdurl": "0.1.2", + "python-multipart": "0.0.32", + "mlx-metal": "0.32.4", + "PyYAML": "6.0.3", + "httpcore2": "2.16.0", + "opencv-python": "5.0.0.93", + "truststore": "0.10.5", + "frozenlist": "1.8.0", + "cffi": "2.1.1", + "httpx2": "2.16.0", + "aiosignal": "1.4.0", + "typer": "0.27.3", + "mlx-audio": "0.6.2", + "Jinja2": "3.1.6", + "annotated-types": "0.8.0", + "regex": "2026.9.29", + "markdown-it-py": "4.2.0", + "tzdata": "2026.6", + "annotated-doc": "0.0.5", + "typing-inspection": "0.4.4", + "fastapi": "0.162.0", + "aiohttp": "3.14.5", + "attrs": "26.1.0", + "multidict": "7.2.1", + "typing_extensions": "4.16.0", + "MarkupSafe": "3.0.4", + "pydantic": "2.14.1", + "python-dateutil": "3.0.0", + "multiprocess": "0.70.19", + "charset-normalizer": "3.5.2", + "anyio": "4.15.1", + "six": "1.17.0", + "scipy": "1.18.1", + "dill": "0.4.1", + "transformers": "5.21.0", + "idna": "3.20" + } + }, + "requirements_sha256": "cdede98457af90bb37dc146204af09dc67059ae7829932d7bae76b9185740ee4", + "models": { + "LFM3-1.7B-MLX-4bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-4bit", + "revision": "d0ca73dcfa70ce18733754a492cdc1fb0123262d", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 2423, + "sha256": "7df2ea819f9a31b770192a0a9dc1b7953a7dd8c41b816cc3905bbec99ef332bd", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 328, + "sha256": "0095ba7158e938b1e2d3082d412f00d8be85e42cbbd7608b63866f7a081cf270", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 973083386, + "sha256": "1be5bf672013cd91aa62148f77c5eea8d2784a95845d5fe826e08c7b84823815", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 50700, + "sha256": "2c8b88e25f6a1055164dd051436b170872200047bce4b3ec5f2a58f6ef929a75", + "verified_against_pinned_revision": true + }, + "special_tokens_map.json": { + "bytes": 92, + "sha256": "7946f864cbfb62a1b041bda937244210a1a87f4a6cab3a48b791131b2cd57f88", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908634, + "sha256": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1859, + "sha256": "b77214b9f12be74b9e5e1dab35d7dace077a31be969314bf38866fd6c8163853", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-VL-2B-MLX-4bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "revision": "c823fade98e35a3d6942ad4d38b1ea15f675cd12", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 4419, + "sha256": "0efb7708e0771e3e104ac628954cdc71070dd436e0b7974615bd1b9eb836f9d9", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 323, + "sha256": "30b1115bcdc4e5671567a5b1255a2f3d2f5a912a79292051c6f1e9db81b5e8c4", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1590959088, + "sha256": "58268dfdf441ae4020d2a52fc97a814645e8a9624946e2d490403d4dcd71a255", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 61256, + "sha256": "38b756dbde167ab7c60756af7d5652414c31afdf4e77700b8d3c99395515d639", + "verified_against_pinned_revision": true + }, + "processor_config.json": { + "bytes": 1254, + "sha256": "f183f5a5e5b3a32e666fce05033e6cb7ca496d26e709a2cee611a13eb7d6b610", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908258, + "sha256": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1759, + "sha256": "d63d1ce2f332047e398764b4170352b96629a5194643105bfd6d70c90b750396", + "verified_against_pinned_revision": true + } + } + } + }, + "client_sha256": { + "$REPO/tools/lfm3/bench_decode.py": "c011106f4f182db19371e3f91f70bcc374160786dd105f2d713a7fea07ab8c4c", + "$REPO/tools/bench_concurrent.py": "0e72e26769f082459329fc068c25f6f72a9155e422962ea54cb5d1226227752d", + "$REPO/tools/bench_evidence.py": "54c28474f0804dbd120e3fb5859be0d1667ef21ca502880d0bed098ec9bb42ff", + "$PACKET/cold_client.py": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2" + }, + "fixture_sha256": { + "vl-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "text-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "text-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "vl-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "text-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "vl-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "vl-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd", + "text-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd" + }, + "packet_files": { + "run_existing_clients.py": "43d10d451a76afe6c895802355e87946f8b1680bbf02cdb215fd9c17a3f9ba05", + "config.json": "0cdeabce95cdaf8273f9e30ea59d12a24a936d526d5f82d3ae0e56c3e95711c1", + "guard.py": "0caf67448d401704bd849744584248729decf5fd5b994f669a518cb61891df37", + "requirements.lock": "cdede98457af90bb37dc146204af09dc67059ae7829932d7bae76b9185740ee4", + "setup-standard.sh": "e77b6ce06eda842a3c30a043148db3b543a4204505ccd77d726781a6d743d41c", + "independent-review.json": "fcea5c48ef445a2c59d386ff03ccafd86d22f4a10a734893ec3d787cd3b118c1", + "combined-source-manifest.json": "2c86619e73c86e6d21a99f1276102294c6c538005af393af5ea4bd6178b61ddc", + "combined-source.patch": "58df0a67e783c6d3af89fe18fb878083c9f1055ed09a692340b13612eaf0d9f2" + } + }, + "protocol": { + "phase": { + "families": [ + "text", + "vl" + ], + "bits": [ + 4 + ], + "cold_tokens": [ + 2048, + 8192 + ], + "decode_tokens": 256, + "cold_output_tokens": 16, + "reps": 3, + "levels": [ + 1, + 4 + ], + "order": [ + "public", + "combined", + "standard", + "standard", + "combined", + "public" + ] + }, + "settings": { + "native_context": 70000, + "parallel": 4, + "prompt_cache_gib": 0, + "temperature": 0, + "thinking": false, + "prefix_cache": "disabled on both", + "port": 18085 + }, + "lock_path": "/private/tmp/tensorfold-model.lock", + "guard_source": { + "origin": "/private/tmp/lfm3-local-locked-profile.py", + "sha256": "0145717ffc373c5cfed39dd11a9943154d9b61d33badef6cd27c4a866e7c6057", + "change": "admission allows only owned descendants while a measured server is running; recognizes frozen binary names" + }, + "admission": "Exclusive flock held throughout model inventory, all twelve sequential owned server legs, measurement and cleanup. Process admission rejects foreign model jobs; only owned server process groups are cleaned up.", + "lock_events": [ + { + "event": "lock_wait", + "time_ns": 1793194653503933000, + "path": "/private/tmp/tensorfold-model.lock" + }, + { + "event": "lock_acquired", + "time_ns": 1793195302533622000, + "path": "/private/tmp/tensorfold-model.lock", + "pid": 47771, + "device": 16777231, + "inode": 46835039 + }, + { + "event": "complete", + "time_ns": 1793195924040963000, + "legs": 12, + "all_lengths_validated": true, + "performance_gate_claim": false + }, + { + "event": "lock_release", + "time_ns": 1793195924041207000, + "pid": 47771 + } + ], + "server_commands": [ + { + "engine": "public", + "family": "text", + "command": [ + "$PACKET/artifacts/tensorfold-native-public", + "serve", + "$MODELS/LFM3-1.7B-MLX-4bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "engine": "combined", + "family": "text", + "command": [ + "$PACKET/artifacts/tensorfold-native-combined", + "serve", + "$MODELS/LFM3-1.7B-MLX-4bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "engine": "standard", + "family": "text", + "command": [ + "$PACKET/standard-venv/bin/python", + "-m", + "mlx_vlm.server", + "--model", + "$MODELS/LFM3-1.7B-MLX-4bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--max-tokens", + "256", + "--max-num-seqs", + "4", + "--max-kv-size", + "70000", + "--vision-cache-size", + "0", + "--log-progress-interval", + "0" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": "70000" + }, + { + "engine": "public", + "family": "vl", + "command": [ + "$PACKET/artifacts/tensorfold-native-public", + "serve", + "$MODELS/LFM3-VL-2B-MLX-4bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "engine": "combined", + "family": "vl", + "command": [ + "$PACKET/artifacts/tensorfold-native-combined", + "serve", + "$MODELS/LFM3-VL-2B-MLX-4bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "engine": "standard", + "family": "vl", + "command": [ + "$PACKET/standard-venv/bin/python", + "-m", + "mlx_vlm.server", + "--model", + "$MODELS/LFM3-VL-2B-MLX-4bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--max-tokens", + "256", + "--max-num-seqs", + "4", + "--max-kv-size", + "70000", + "--vision-cache-size", + "0", + "--log-progress-interval", + "0" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": "70000" + } + ], + "memory_policy": "Native admission unchanged; MLX memory and wired limits unmodified. /usr/bin/time -l captures physical footprint and RSS; available server metrics preserved. No allocator/pinned peak inferred from RSS.", + "replay": [ + "python3 \"$PACKET/run_existing_clients.py\"", + "python3 \"$PACKET/run_existing_clients.py\" --execute --output \"$NEW_OUTPUT\"" + ], + "replay_requirements": "Original frozen packet and artifacts are local retained inputs, not bundled executable tooling in this receipt. Set REPO/MODELS/PACKET paths appropriately; output must not exist. Packet validates pinned dependencies, binaries, client/fixture and model files before load. Reconstruct from the hashes/versions/commands here if the retained packet is unavailable." + }, + "results": { + "legs": [ + { + "family": "text", + "bits": 4, + "leg": 0, + "engine": "public", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 3.1976742083206773, + "output_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 13.849388832692057, + "output_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1217773568, + "peak_physical_footprint_bytes": 1514243752 + }, + "tag": "text-4-0-public", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 175.0, + "per_stream_tps": 175.0, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 166.59344765496334, + "makespan_s": 1.5366750829853117, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 174.33357930951772, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5420032078400254, + "end_to_end_tps": 166.01781286732486, + "timing_valid": true, + "per_stream_tps": [ + 174.3 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 176.54645032063524, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5256117079406977, + "end_to_end_tps": 167.80154391024837, + "timing_valid": true, + "per_stream_tps": [ + 176.5 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 175.0280052102641, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5366750829853117, + "end_to_end_tps": 166.59344765496334, + "timing_valid": true, + "per_stream_tps": [ + 175.0 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 388.3, + "per_stream_tps": 102.1, + "ttft_s_max": 0.32, + "timing_valid": true, + "end_to_end_tps": 378.04254569938155, + "makespan_s": 2.708689833059907, + "steady_tps": 428.1, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 388.32442956935364, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.708689833059907, + "end_to_end_tps": 378.04254569938155, + "timing_valid": true, + "steady_tps": 428.07189807812284, + "per_stream_tps": [ + 97.3, + 103.4, + 106.9, + 100.4 + ], + "ttft_s": [ + 0.08, + 0.24, + 0.32, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 387.38800732228174, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.714653834234923, + "end_to_end_tps": 377.2119992192655, + "timing_valid": true, + "steady_tps": 426.9452635399727, + "per_stream_tps": [ + 103.2, + 106.6, + 97.1, + 100.2 + ], + "ttft_s": [ + 0.24, + 0.32, + 0.08, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 390.4867598842693, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.6939699589274824, + "end_to_end_tps": 380.1081732951739, + "timing_valid": true, + "steady_tps": 430.80266743672485, + "per_stream_tps": [ + 97.9, + 101.0, + 107.6, + 104.1 + ], + "ttft_s": [ + 0.08, + 0.16, + 0.32, + 0.24 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 183.7, + "per_stream_tps": 183.7, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 174.05589265951394, + "makespan_s": 1.4707919168286026, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 183.7156213823559, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.4707919168286026, + "end_to_end_tps": 174.05589265951394, + "timing_valid": true, + "per_stream_tps": [ + 183.7 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 183.29057256770892, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.4716297080740333, + "end_to_end_tps": 173.95680353248306, + "timing_valid": true, + "per_stream_tps": [ + 183.3 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 184.09310995785592, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.4658530410379171, + "end_to_end_tps": 174.64233646418998, + "timing_valid": true, + "per_stream_tps": [ + 184.1 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 393.1, + "per_stream_tps": 103.5, + "ttft_s_max": 0.32, + "timing_valid": true, + "end_to_end_tps": 382.64031739157696, + "makespan_s": 2.676142459269613, + "steady_tps": 434.9, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 392.42369639073456, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.680690708104521, + "end_to_end_tps": 381.9911028542551, + "timing_valid": true, + "steady_tps": 433.9434805978308, + "per_stream_tps": [ + 108.1, + 104.5, + 98.6, + 101.8 + ], + "ttft_s": [ + 0.32, + 0.24, + 0.08, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 393.0500466804608, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.676142459269613, + "end_to_end_tps": 382.64031739157696, + "timing_valid": true, + "steady_tps": 434.892181071932, + "per_stream_tps": [ + 104.7, + 108.3, + 101.9, + 98.8 + ], + "ttft_s": [ + 0.24, + 0.32, + 0.16, + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 395.57580143014764, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.6601654160767794, + "end_to_end_tps": 384.938468040908, + "timing_valid": true, + "steady_tps": 437.80964906628566, + "per_stream_tps": [ + 102.6, + 109.0, + 105.4, + 99.4 + ], + "ttft_s": [ + 0.16, + 0.32, + 0.24, + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + } + ] + } + ] + }, + { + "family": "text", + "bits": 4, + "leg": 1, + "engine": "combined", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 3.4472638331353664, + "output_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 17.29660391714424, + "output_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1218560000, + "peak_physical_footprint_bytes": 1514964696 + }, + "tag": "text-4-1-combined", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 166.1, + "per_stream_tps": 166.1, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 158.45870434113, + "makespan_s": 1.61556287528947, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 166.0869678921436, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.6169291250407696, + "end_to_end_tps": 158.32481216116702, + "timing_valid": true, + "per_stream_tps": [ + 166.1 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 166.11623932730092, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.61556287528947, + "end_to_end_tps": 158.45870434113, + "timing_valid": true, + "per_stream_tps": [ + 166.1 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 172.19134049652038, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5611517080105841, + "end_to_end_tps": 163.98150076409127, + "timing_valid": true, + "per_stream_tps": [ + 172.2 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 383.5, + "per_stream_tps": 100.3, + "ttft_s_max": 0.32, + "timing_valid": true, + "end_to_end_tps": 373.6487502745828, + "makespan_s": 2.74054175009951, + "steady_tps": 420.9, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 382.07900870977994, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.7496890421025455, + "end_to_end_tps": 372.40574636650547, + "timing_valid": true, + "steady_tps": 419.1548067080006, + "per_stream_tps": [ + 98.5, + 95.6, + 104.9, + 101.6 + ], + "ttft_s": [ + 0.16, + 0.08, + 0.32, + 0.24 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 383.5114695784525, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.74054175009951, + "end_to_end_tps": 373.6487502745828, + "timing_valid": true, + "steady_tps": 420.89965263046037, + "per_stream_tps": [ + 98.9, + 105.3, + 102.0, + 95.9 + ], + "ttft_s": [ + 0.16, + 0.32, + 0.24, + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 384.30602814425293, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.734592542052269, + "end_to_end_tps": 374.46163706403735, + "timing_valid": true, + "steady_tps": 421.5933306596878, + "per_stream_tps": [ + 96.1, + 99.0, + 102.2, + 105.5 + ], + "ttft_s": [ + 0.08, + 0.16, + 0.24, + 0.32 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 177.9, + "per_stream_tps": 177.9, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 169.02450905280594, + "makespan_s": 1.5145732499659061, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 177.79868486495323, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5152165829204023, + "end_to_end_tps": 168.95274437043847, + "timing_valid": true, + "per_stream_tps": [ + 177.8 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 178.15170226681155, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5122410408221185, + "end_to_end_tps": 169.2851821167527, + "timing_valid": true, + "per_stream_tps": [ + 178.2 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 177.90131392071672, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5145732499659061, + "end_to_end_tps": 169.02450905280594, + "timing_valid": true, + "per_stream_tps": [ + 177.9 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 386.5, + "per_stream_tps": 101.2, + "ttft_s_max": 0.32, + "timing_valid": true, + "end_to_end_tps": 376.30745827433486, + "makespan_s": 2.7211791248992085, + "steady_tps": 424.9, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 386.51572926730825, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.7211791248992085, + "end_to_end_tps": 376.30745827433486, + "timing_valid": true, + "steady_tps": 425.179225013546, + "per_stream_tps": [ + 106.4, + 99.7, + 102.9, + 96.6 + ], + "ttft_s": [ + 0.32, + 0.16, + 0.24, + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 385.85576947303315, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.7254131245426834, + "end_to_end_tps": 375.7228549238106, + "timing_valid": true, + "steady_tps": 424.16428908318574, + "per_stream_tps": [ + 96.5, + 99.5, + 106.2, + 102.7 + ], + "ttft_s": [ + 0.08, + 0.16, + 0.32, + 0.24 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 386.5338735839194, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.7206580420024693, + "end_to_end_tps": 376.37953178647604, + "timing_valid": true, + "steady_tps": 424.9444810729043, + "per_stream_tps": [ + 106.3, + 99.7, + 102.9, + 96.6 + ], + "ttft_s": [ + 0.32, + 0.16, + 0.24, + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + } + ] + } + ] + }, + { + "family": "text", + "bits": 4, + "leg": 2, + "engine": "standard", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 1.4001411660574377, + "output_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "request_body_sha256": "f4de9e20df38afd2c3afa19dc623f6d6265fb5d387acefa1ec46c84d806a7807" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.058180375024676, + "output_sha256": "3855897e8f18058e244110a049fbe9ab9131d137fd6b116ec4d426c83093a1b9", + "request_body_sha256": "0ddc26bb0891cd0c2bd0d37b9fe6d431e724e8dc001873b3b6a487c6c52157a2" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "unverified if server omits token hashes; text digest is not substituted" + }, + "memory": { + "max_rss_bytes": 1509490688, + "peak_physical_footprint_bytes": 2763966528 + }, + "tag": "text-4-2-standard", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 169.8, + "per_stream_tps": 169.8, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 165.148091790501, + "makespan_s": 1.550123875029385, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 172.26463880154267, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.527306083124131, + "end_to_end_tps": 167.615386875398, + "timing_valid": true, + "per_stream_tps": [ + 172.3 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 168.81678185058905, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5574545841664076, + "end_to_end_tps": 164.37076406758803, + "timing_valid": true, + "per_stream_tps": [ + 168.8 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 169.78287602547306, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.550123875029385, + "end_to_end_tps": 165.148091790501, + "timing_valid": true, + "per_stream_tps": [ + 169.8 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + null + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 429.2, + "per_stream_tps": 109.8, + "ttft_s_max": 0.2, + "timing_valid": true, + "end_to_end_tps": 413.82319663768277, + "makespan_s": 2.4744867091067135, + "steady_tps": 442.2, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 446.02964748705443, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.407312999945134, + "end_to_end_tps": 425.37052723237, + "timing_valid": true, + "steady_tps": 470.87674116674157, + "per_stream_tps": [ + 113.8, + 115.7, + 115.7, + 115.7 + ], + "ttft_s": [ + 0.12, + 0.2, + 0.2, + 0.2 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 429.1684099599845, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.4744867091067135, + "end_to_end_tps": 413.82319663768277, + "timing_valid": true, + "steady_tps": 442.1819629857103, + "per_stream_tps": [ + 109.8, + 109.8, + 109.8, + 108.4 + ], + "ttft_s": [ + 0.15, + 0.15, + 0.15, + 0.1 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 361.63599877811157, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.950282083824277, + "end_to_end_tps": 347.0854551889659, + "timing_valid": true, + "steady_tps": 367.8956362305029, + "per_stream_tps": [ + 92.0, + 90.9, + 90.9, + 92.0 + ], + "ttft_s": [ + 0.18, + 0.13, + 0.13, + 0.18 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 171.4, + "per_stream_tps": 171.4, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 166.26456297584537, + "makespan_s": 1.5397147499024868, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 171.61466519462007, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.538818625267595, + "end_to_end_tps": 166.3613864535091, + "timing_valid": true, + "per_stream_tps": [ + 171.6 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "a7876bd7a6b92868f6dd64f5434b8c2f0fec52b1aa804597d22dd119a43e4893" + } + ], + "equal_alone": [ + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 171.44383317187626, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5397147499024868, + "end_to_end_tps": 166.26456297584537, + "timing_valid": true, + "per_stream_tps": [ + 171.4 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "a7876bd7a6b92868f6dd64f5434b8c2f0fec52b1aa804597d22dd119a43e4893" + } + ], + "equal_alone": [ + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 170.28580590814167, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5503849172964692, + "end_to_end_tps": 165.12028538461777, + "timing_valid": true, + "per_stream_tps": [ + 170.3 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "a7876bd7a6b92868f6dd64f5434b8c2f0fec52b1aa804597d22dd119a43e4893" + } + ], + "equal_alone": [ + null + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 483.7, + "per_stream_tps": 121.8, + "ttft_s_max": 0.16, + "timing_valid": true, + "end_to_end_tps": 454.5412089186243, + "makespan_s": 2.252821042202413, + "steady_tps": 486.2, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 484.0013914306858, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.252148542087525, + "end_to_end_tps": 454.676936651279, + "timing_valid": true, + "steady_tps": 486.60997934418856, + "per_stream_tps": [ + 121.9, + 121.9, + 121.8, + 121.9 + ], + "ttft_s": [ + 0.16, + 0.16, + 0.14, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f4bf4c61241b482a3f03af29627476511ba225bd5c64d363e40dfca30125ee5" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 483.69557655279243, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.252821042202413, + "end_to_end_tps": 454.5412089186243, + "timing_valid": true, + "steady_tps": 486.1704069494632, + "per_stream_tps": [ + 121.8, + 121.8, + 121.8, + 121.8 + ], + "ttft_s": [ + 0.16, + 0.16, + 0.16, + 0.14 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f4bf4c61241b482a3f03af29627476511ba225bd5c64d363e40dfca30125ee5" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 472.22991978123383, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.30667491722852, + "end_to_end_tps": 443.92904797800486, + "timing_valid": true, + "steady_tps": 474.7450476909172, + "per_stream_tps": [ + 118.9, + 118.9, + 118.9, + 118.9 + ], + "ttft_s": [ + 0.16, + 0.16, + 0.16, + 0.15 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f4bf4c61241b482a3f03af29627476511ba225bd5c64d363e40dfca30125ee5" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + } + ] + } + ] + }, + { + "family": "text", + "bits": 4, + "leg": 3, + "engine": "standard", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 1.0871214168146253, + "output_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "request_body_sha256": "f4de9e20df38afd2c3afa19dc623f6d6265fb5d387acefa1ec46c84d806a7807" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.053703292272985, + "output_sha256": "3855897e8f18058e244110a049fbe9ab9131d137fd6b116ec4d426c83093a1b9", + "request_body_sha256": "0ddc26bb0891cd0c2bd0d37b9fe6d431e724e8dc001873b3b6a487c6c52157a2" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "unverified if server omits token hashes; text digest is not substituted" + }, + "memory": { + "max_rss_bytes": 1487306752, + "peak_physical_footprint_bytes": 2758903896 + }, + "tag": "text-4-3-standard", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 172.4, + "per_stream_tps": 172.4, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 167.58153712663864, + "makespan_s": 1.527614583261311, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 174.06198297170022, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5120207923464477, + "end_to_end_tps": 169.3098410391059, + "timing_valid": true, + "per_stream_tps": [ + 174.1 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 172.36259324598905, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.527614583261311, + "end_to_end_tps": 167.58153712663864, + "timing_valid": true, + "per_stream_tps": [ + 172.4 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 171.4222667734278, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5342487921006978, + "end_to_end_tps": 166.85690177372348, + "timing_valid": true, + "per_stream_tps": [ + 171.4 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + null + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 472.0, + "per_stream_tps": 120.7, + "ttft_s_max": 0.17, + "timing_valid": true, + "end_to_end_tps": 453.47250495414346, + "makespan_s": 2.258130291942507, + "steady_tps": 485.8, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 472.1309561767671, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.258130291942507, + "end_to_end_tps": 453.47250495414346, + "timing_valid": true, + "steady_tps": 486.27637219299885, + "per_stream_tps": [ + 121.3, + 118.9, + 121.3, + 121.3 + ], + "ttft_s": [ + 0.15, + 0.1, + 0.15, + 0.15 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 471.99612113868454, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.2572577078826725, + "end_to_end_tps": 453.6478030062952, + "timing_valid": true, + "steady_tps": 485.82773198364686, + "per_stream_tps": [ + 121.2, + 118.8, + 121.2, + 121.2 + ], + "ttft_s": [ + 0.15, + 0.1, + 0.15, + 0.15 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 469.87623090402326, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.2895848751068115, + "end_to_end_tps": 447.2426469677082, + "timing_valid": true, + "steady_tps": 481.33774288695304, + "per_stream_tps": [ + 120.2, + 118.2, + 118.2, + 120.2 + ], + "ttft_s": [ + 0.17, + 0.12, + 0.12, + 0.17 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 173.1, + "per_stream_tps": 173.1, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 167.71992538792222, + "makespan_s": 1.5263541252352297, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 173.0794732633171, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5263541252352297, + "end_to_end_tps": 167.71992538792222, + "timing_valid": true, + "per_stream_tps": [ + 173.1 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "a7876bd7a6b92868f6dd64f5434b8c2f0fec52b1aa804597d22dd119a43e4893" + } + ], + "equal_alone": [ + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 173.45309958433361, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.52307637501508, + "end_to_end_tps": 168.0808685627898, + "timing_valid": true, + "per_stream_tps": [ + 173.5 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "a7876bd7a6b92868f6dd64f5434b8c2f0fec52b1aa804597d22dd119a43e4893" + } + ], + "equal_alone": [ + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 172.69256971026414, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5297713750042021, + "end_to_end_tps": 167.34526752358457, + "timing_valid": true, + "per_stream_tps": [ + 172.7 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "a7876bd7a6b92868f6dd64f5434b8c2f0fec52b1aa804597d22dd119a43e4893" + } + ], + "equal_alone": [ + null + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 473.6, + "per_stream_tps": 119.2, + "ttft_s_max": 0.18, + "timing_valid": true, + "end_to_end_tps": 445.49182875323265, + "makespan_s": 2.298583125229925, + "steady_tps": 476.2, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 473.58003752344314, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.298583125229925, + "end_to_end_tps": 445.49182875323265, + "timing_valid": true, + "steady_tps": 476.18323085090094, + "per_stream_tps": [ + 119.2, + 119.3, + 119.2, + 119.2 + ], + "ttft_s": [ + 0.16, + 0.14, + 0.16, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f4bf4c61241b482a3f03af29627476511ba225bd5c64d363e40dfca30125ee5" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 451.4560536062305, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.4141651657409966, + "end_to_end_tps": 424.1631908750106, + "timing_valid": true, + "steady_tps": 453.75615253180536, + "per_stream_tps": [ + 113.7, + 113.6, + 113.7, + 113.7 + ], + "ttft_s": [ + 0.17, + 0.15, + 0.17, + 0.17 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f4bf4c61241b482a3f03af29627476511ba225bd5c64d363e40dfca30125ee5" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 479.2751294144382, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.29608166590333, + "end_to_end_tps": 445.97716849811417, + "timing_valid": true, + "steady_tps": 481.7707850826629, + "per_stream_tps": [ + 120.7, + 120.6, + 120.8, + 120.6 + ], + "ttft_s": [ + 0.18, + 0.17, + 0.18, + 0.17 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "5f2fb4e13813e05247efda099109b8a0aef646afce9489033406ab2bcdd6110c" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + } + ] + } + ] + }, + { + "family": "text", + "bits": 4, + "leg": 4, + "engine": "combined", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 3.3981029582209885, + "output_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 17.221552750095725, + "output_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1217888256, + "peak_physical_footprint_bytes": 1514325720 + }, + "tag": "text-4-4-combined", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 169.4, + "per_stream_tps": 169.4, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 161.47914897903064, + "makespan_s": 1.5853440002538264, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 168.65410283796538, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.592514291871339, + "end_to_end_tps": 160.75208951448613, + "timing_valid": true, + "per_stream_tps": [ + 168.7 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 169.3786485536357, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5853440002538264, + "end_to_end_tps": 161.47914897903064, + "timing_valid": true, + "per_stream_tps": [ + 169.4 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 169.78297962149645, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5819875411689281, + "end_to_end_tps": 161.8217548103078, + "timing_valid": true, + "per_stream_tps": [ + 169.8 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 380.4, + "per_stream_tps": 99.0, + "ttft_s_max": 0.32, + "timing_valid": true, + "end_to_end_tps": 370.7111857287613, + "makespan_s": 2.7622581659816206, + "steady_tps": 417.1, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 381.6083183952793, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.753275915980339, + "end_to_end_tps": 371.92058887254376, + "timing_valid": true, + "steady_tps": 418.40427063797677, + "per_stream_tps": [ + 104.7, + 101.4, + 98.3, + 95.4 + ], + "ttft_s": [ + 0.32, + 0.24, + 0.16, + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 380.37264712928885, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.7622581659816206, + "end_to_end_tps": 370.7111857287613, + "timing_valid": true, + "steady_tps": 417.08940388030396, + "per_stream_tps": [ + 104.4, + 101.1, + 95.1, + 98.0 + ], + "ttft_s": [ + 0.32, + 0.24, + 0.08, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 374.9253882115582, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.8036270420998335, + "end_to_end_tps": 365.24116247396955, + "timing_valid": true, + "steady_tps": 411.02231191279714, + "per_stream_tps": [ + 96.7, + 99.7, + 93.8, + 102.8 + ], + "ttft_s": [ + 0.16, + 0.24, + 0.08, + 0.32 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 175.0, + "per_stream_tps": 175.0, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 166.43311669134442, + "makespan_s": 1.538155416958034, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 174.98129958626902, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.538155416958034, + "end_to_end_tps": 166.43311669134442, + "timing_valid": true, + "per_stream_tps": [ + 175.0 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 175.40580218595554, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5350151252932847, + "end_to_end_tps": 166.77360097744173, + "timing_valid": true, + "per_stream_tps": [ + 175.4 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 174.17910834276157, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5451102079823613, + "end_to_end_tps": 165.6839743064609, + "timing_valid": true, + "per_stream_tps": [ + 174.2 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 382.1, + "per_stream_tps": 100.0, + "ttft_s_max": 0.32, + "timing_valid": true, + "end_to_end_tps": 372.1148042883409, + "makespan_s": 2.751838916912675, + "steady_tps": 419.6, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 382.5669552494983, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.7473174589686096, + "end_to_end_tps": 372.72722038625534, + "timing_valid": true, + "steady_tps": 420.1929874826971, + "per_stream_tps": [ + 98.6, + 101.8, + 95.7, + 105.1 + ], + "ttft_s": [ + 0.16, + 0.24, + 0.08, + 0.32 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 382.1196774371716, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.751838916912675, + "end_to_end_tps": 372.1148042883409, + "timing_valid": true, + "steady_tps": 419.6302715357268, + "per_stream_tps": [ + 98.5, + 101.7, + 105.0, + 95.5 + ], + "ttft_s": [ + 0.16, + 0.24, + 0.32, + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 381.266970720118, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.7580740000121295, + "end_to_end_tps": 371.27357713951716, + "timing_valid": true, + "steady_tps": 418.7035727124425, + "per_stream_tps": [ + 104.8, + 98.3, + 101.4, + 95.3 + ], + "ttft_s": [ + 0.32, + 0.16, + 0.24, + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + } + ] + } + ] + }, + { + "family": "text", + "bits": 4, + "leg": 5, + "engine": "public", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 3.1604983750730753, + "output_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 14.409123916644603, + "output_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1217822720, + "peak_physical_footprint_bytes": 1514260112 + }, + "tag": "text-4-5-public", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 175.1, + "per_stream_tps": 175.1, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 166.58977074916976, + "makespan_s": 1.536708999890834, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 175.0803803635659, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.536708999890834, + "end_to_end_tps": 166.58977074916976, + "timing_valid": true, + "per_stream_tps": [ + 175.1 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 174.57396389141493, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5404117917641997, + "end_to_end_tps": 166.1893276646557, + "timing_valid": true, + "per_stream_tps": [ + 174.6 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 176.4727407119902, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5246990420855582, + "end_to_end_tps": 167.90198782431884, + "timing_valid": true, + "per_stream_tps": [ + 176.5 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 389.8, + "per_stream_tps": 101.8, + "ttft_s_max": 0.33, + "timing_valid": true, + "end_to_end_tps": 379.7330406581631, + "makespan_s": 2.6966312918812037, + "steady_tps": 429.1, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 385.24300236921965, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.7309770826250315, + "end_to_end_tps": 374.9573757007602, + "timing_valid": true, + "steady_tps": 424.4451267210408, + "per_stream_tps": [ + 102.6, + 106.0, + 99.7, + 96.6 + ], + "ttft_s": [ + 0.25, + 0.33, + 0.17, + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 390.4341652635153, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.6922058328054845, + "end_to_end_tps": 380.3572473999559, + "timing_valid": true, + "steady_tps": 429.45828180855085, + "per_stream_tps": [ + 107.3, + 97.9, + 103.8, + 100.9 + ], + "ttft_s": [ + 0.32, + 0.08, + 0.24, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 389.79263659668885, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.6966312918812037, + "end_to_end_tps": 379.7330406581631, + "timing_valid": true, + "steady_tps": 429.0837043986373, + "per_stream_tps": [ + 103.8, + 97.7, + 107.2, + 100.8 + ], + "ttft_s": [ + 0.24, + 0.08, + 0.32, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 156.7, + "per_stream_tps": 156.7, + "ttft_s_max": 0.09, + "timing_valid": true, + "end_to_end_tps": 149.867731543309, + "makespan_s": 1.708172915969044, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 143.38614873922572, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.8687005839310586, + "end_to_end_tps": 136.993589128907, + "timing_valid": true, + "per_stream_tps": [ + 143.4 + ], + "ttft_s": [ + 0.09 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 156.69145080631193, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.708172915969044, + "end_to_end_tps": 149.867731543309, + "timing_valid": true, + "per_stream_tps": [ + 156.7 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 176.18895002172795, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.5291710831224918, + "end_to_end_tps": 167.41096063447696, + "timing_valid": true, + "per_stream_tps": [ + 176.2 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 333.2, + "per_stream_tps": 88.0, + "ttft_s_max": 0.39, + "timing_valid": true, + "end_to_end_tps": 325.35595639564804, + "makespan_s": 3.147322124801576, + "steady_tps": 367.5, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 333.2213611523689, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 3.147322124801576, + "end_to_end_tps": 325.35595639564804, + "timing_valid": true, + "steady_tps": 365.3434952241035, + "per_stream_tps": [ + 85.8, + 88.4, + 83.3, + 91.4 + ], + "ttft_s": [ + 0.17, + 0.26, + 0.09, + 0.36 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 332.85472325385604, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 3.1585415829904377, + "end_to_end_tps": 324.2002592318254, + "timing_valid": true, + "steady_tps": 367.50564892719797, + "per_stream_tps": [ + 88.8, + 92.0, + 85.8, + 83.2 + ], + "ttft_s": [ + 0.28, + 0.38, + 0.19, + 0.09 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 338.5504679373183, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 3.110549457836896, + "end_to_end_tps": 329.2022884960327, + "timing_valid": true, + "steady_tps": 375.24274186205247, + "per_stream_tps": [ + 93.9, + 90.6, + 84.7, + 87.5 + ], + "ttft_s": [ + 0.39, + 0.29, + 0.1, + 0.2 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + } + ] + } + ] + }, + { + "family": "vl", + "bits": 4, + "leg": 0, + "engine": "public", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.068471458274871, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 14.948924167081714, + "output_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1835630592, + "peak_physical_footprint_bytes": 2173503600 + }, + "tag": "vl-4-0-public", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 152.1, + "per_stream_tps": 152.1, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 145.73308401864628, + "makespan_s": 1.756636125035584, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 152.05740745566536, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.756636125035584, + "end_to_end_tps": 145.73308401864628, + "timing_valid": true, + "per_stream_tps": [ + 152.1 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 152.6768289446356, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.7505523329600692, + "end_to_end_tps": 146.23955832678294, + "timing_valid": true, + "per_stream_tps": [ + 152.7 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 152.10096918029834, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.7571592084132135, + "end_to_end_tps": 145.6897011803378, + "timing_valid": true, + "per_stream_tps": [ + 152.1 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 379.8, + "per_stream_tps": 96.7, + "ttft_s_max": 0.34, + "timing_valid": true, + "end_to_end_tps": 370.0728963147554, + "makespan_s": 2.7670224169269204, + "steady_tps": 415.8, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 382.4665220808524, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.7486733752302825, + "end_to_end_tps": 372.54335463347286, + "timing_valid": true, + "steady_tps": 419.0713640476002, + "per_stream_tps": [ + 95.6, + 98.6, + 105.1, + 101.7 + ], + "ttft_s": [ + 0.08, + 0.16, + 0.32, + 0.24 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 379.75781631994755, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.7670224169269204, + "end_to_end_tps": 370.0728963147554, + "timing_valid": true, + "steady_tps": 415.7703274806987, + "per_stream_tps": [ + 100.9, + 95.0, + 104.3, + 97.8 + ], + "ttft_s": [ + 0.24, + 0.08, + 0.32, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 343.2818701092185, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 3.053055041935295, + "end_to_end_tps": 335.40174871885006, + "timing_valid": true, + "steady_tps": 374.7521344537325, + "per_stream_tps": [ + 88.4, + 91.1, + 85.8, + 94.0 + ], + "ttft_s": [ + 0.17, + 0.25, + 0.08, + 0.34 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 150.5, + "per_stream_tps": 150.5, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 144.03289574580631, + "makespan_s": 1.7773717502132058, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 148.1346901549664, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.8062679171562195, + "end_to_end_tps": 141.7286979237528, + "timing_valid": true, + "per_stream_tps": [ + 148.1 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 150.4833021557889, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.7773717502132058, + "end_to_end_tps": 144.03289574580631, + "timing_valid": true, + "per_stream_tps": [ + 150.5 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 152.05146118191536, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.7596968747675419, + "end_to_end_tps": 145.4796014420483, + "timing_valid": true, + "per_stream_tps": [ + 152.1 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 382.6, + "per_stream_tps": 100.0, + "ttft_s_max": 0.33, + "timing_valid": true, + "end_to_end_tps": 372.62119661975504, + "makespan_s": 2.7480991668999195, + "steady_tps": 420.4, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 384.2723782007661, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.736787499859929, + "end_to_end_tps": 374.1613114106993, + "timing_valid": true, + "steady_tps": 421.7314494193366, + "per_stream_tps": [ + 99.1, + 105.7, + 96.1, + 102.3 + ], + "ttft_s": [ + 0.16, + 0.32, + 0.08, + 0.24 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 382.5557575031354, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.7480991668999195, + "end_to_end_tps": 372.62119661975504, + "timing_valid": true, + "steady_tps": 420.3756218434952, + "per_stream_tps": [ + 105.3, + 95.7, + 101.9, + 98.6 + ], + "ttft_s": [ + 0.33, + 0.08, + 0.25, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 379.8426799933923, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.7681425409391522, + "end_to_end_tps": 369.9231469679252, + "timing_valid": true, + "steady_tps": 416.8617845698784, + "per_stream_tps": [ + 95.0, + 97.9, + 104.5, + 101.0 + ], + "ttft_s": [ + 0.08, + 0.16, + 0.33, + 0.24 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + } + ] + } + ] + }, + { + "family": "vl", + "bits": 4, + "leg": 1, + "engine": "combined", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 3.4114462919533253, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 17.34155425010249, + "output_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1836417024, + "peak_physical_footprint_bytes": 2174273720 + }, + "tag": "vl-4-1-combined", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 146.2, + "per_stream_tps": 146.2, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 140.31503239974086, + "makespan_s": 1.8244659579358995, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 146.20333234391993, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.8244659579358995, + "end_to_end_tps": 140.31503239974086, + "timing_valid": true, + "per_stream_tps": [ + 146.2 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 146.5534788207802, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.8206167910248041, + "end_to_end_tps": 140.61168789721015, + "timing_valid": true, + "per_stream_tps": [ + 146.6 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 139.5312984437731, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.908918417058885, + "end_to_end_tps": 134.1073550929563, + "timing_valid": true, + "per_stream_tps": [ + 139.5 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 372.0, + "per_stream_tps": 97.6, + "ttft_s_max": 0.32, + "timing_valid": true, + "end_to_end_tps": 362.62879488760325, + "makespan_s": 2.823824292048812, + "steady_tps": 406.6, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 373.2887956221813, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.813908000010997, + "end_to_end_tps": 363.90670910207376, + "timing_valid": true, + "steady_tps": 407.92066981152647, + "per_stream_tps": [ + 93.6, + 102.3, + 96.4, + 99.1 + ], + "ttft_s": [ + 0.08, + 0.32, + 0.16, + 0.24 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 371.9627317535548, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.823824292048812, + "end_to_end_tps": 362.62879488760325, + "timing_valid": true, + "steady_tps": 406.6010371400546, + "per_stream_tps": [ + 96.1, + 101.9, + 98.8, + 93.3 + ], + "ttft_s": [ + 0.16, + 0.32, + 0.24, + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 371.86516750959413, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.824336875230074, + "end_to_end_tps": 362.56298212180644, + "timing_valid": true, + "steady_tps": 406.29939084344124, + "per_stream_tps": [ + 98.7, + 93.2, + 101.9, + 96.1 + ], + "ttft_s": [ + 0.24, + 0.08, + 0.32, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 146.1, + "per_stream_tps": 146.1, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 140.13440133587764, + "makespan_s": 1.8268176661804318, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 146.13235292633067, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.8268176661804318, + "end_to_end_tps": 140.13440133587764, + "timing_valid": true, + "per_stream_tps": [ + 146.1 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 146.63771935020262, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.820876999758184, + "end_to_end_tps": 140.59159406922998, + "timing_valid": true, + "per_stream_tps": [ + 146.6 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 145.76576185599458, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.831446791999042, + "end_to_end_tps": 139.78020061427694, + "timing_valid": true, + "per_stream_tps": [ + 145.8 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 372.0, + "per_stream_tps": 97.7, + "ttft_s_max": 0.33, + "timing_valid": true, + "end_to_end_tps": 362.31185392419746, + "makespan_s": 2.826294499915093, + "steady_tps": 408.5, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 374.5021496466539, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.80641687521711, + "end_to_end_tps": 364.8780796048988, + "timing_valid": true, + "steady_tps": 411.45323104051005, + "per_stream_tps": [ + 96.8, + 93.9, + 102.9, + 99.6 + ], + "ttft_s": [ + 0.16, + 0.08, + 0.33, + 0.25 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 371.0655850940012, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.8315105410292745, + "end_to_end_tps": 361.64442447308306, + "timing_valid": true, + "steady_tps": 407.4806092059769, + "per_stream_tps": [ + 101.8, + 98.6, + 95.9, + 93.1 + ], + "ttft_s": [ + 0.33, + 0.25, + 0.16, + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 371.9541637702016, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.826294499915093, + "end_to_end_tps": 362.31185392419746, + "timing_valid": true, + "steady_tps": 408.48259351888174, + "per_stream_tps": [ + 102.1, + 93.3, + 98.9, + 96.2 + ], + "ttft_s": [ + 0.33, + 0.08, + 0.25, + 0.17 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + } + ] + } + ] + }, + { + "family": "vl", + "bits": 4, + "leg": 2, + "engine": "standard", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 1.0534550421871245, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "025e7799abd6977ff0c3185fc32ee75fbd4ba3d404535565ec46fae4915b795f" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.1006536250934005, + "output_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "request_body_sha256": "ed436d1be8120ed043052d1801596681eb49c393fba11ef39b12b6ee94487bc5" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "unverified if server omits token hashes; text digest is not substituted" + }, + "memory": { + "max_rss_bytes": 2108932096, + "peak_physical_footprint_bytes": 3283257768 + }, + "tag": "vl-4-2-standard", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 145.4, + "per_stream_tps": 145.4, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 141.9871480600231, + "makespan_s": 1.802980083040893, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 145.39878073480114, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.802980083040893, + "end_to_end_tps": 141.9871480600231, + "timing_valid": true, + "per_stream_tps": [ + 145.4 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + } + ], + "equal_alone": [ + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 145.48893022521824, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.8014503330923617, + "end_to_end_tps": 142.10772026145818, + "timing_valid": true, + "per_stream_tps": [ + 145.5 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + } + ], + "equal_alone": [ + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 141.39606992674067, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.8521992503665388, + "end_to_end_tps": 138.21407170386186, + "timing_valid": true, + "per_stream_tps": [ + 141.4 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + } + ], + "equal_alone": [ + null + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 436.3, + "per_stream_tps": 113.1, + "ttft_s_max": 0.25, + "timing_valid": true, + "end_to_end_tps": 420.0062284206021, + "makespan_s": 2.4380590827204287, + "steady_tps": 451.2, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 425.2857757503177, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.5043137497268617, + "end_to_end_tps": 408.89445266659766, + "timing_valid": true, + "steady_tps": 451.1829800475416, + "per_stream_tps": [ + 113.2, + 113.2, + 113.2, + 107.1 + ], + "ttft_s": [ + 0.25, + 0.25, + 0.25, + 0.11 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9f0cfbcef69a73bf24c3254a60473fbf23d095540107ffbc5bb1d0300ebbec6a" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9f0cfbcef69a73bf24c3254a60473fbf23d095540107ffbc5bb1d0300ebbec6a" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9f0cfbcef69a73bf24c3254a60473fbf23d095540107ffbc5bb1d0300ebbec6a" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "3075eca197d4bdb7803275b0e8b52ca7fece2e4f5690e96237c7de6b4daff334" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 436.27337210105026, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.4380590827204287, + "end_to_end_tps": 420.0062284206021, + "timing_valid": true, + "steady_tps": 446.38024741419105, + "per_stream_tps": [ + 112.0, + 112.0, + 112.0, + 109.9 + ], + "ttft_s": [ + 0.16, + 0.16, + 0.16, + 0.1 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9f0cfbcef69a73bf24c3254a60473fbf23d095540107ffbc5bb1d0300ebbec6a" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9f0cfbcef69a73bf24c3254a60473fbf23d095540107ffbc5bb1d0300ebbec6a" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9f0cfbcef69a73bf24c3254a60473fbf23d095540107ffbc5bb1d0300ebbec6a" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "3075eca197d4bdb7803275b0e8b52ca7fece2e4f5690e96237c7de6b4daff334" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 448.9789523933043, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.3726206249557436, + "end_to_end_tps": 431.5902800596706, + "timing_valid": true, + "steady_tps": 459.0887415691047, + "per_stream_tps": [ + 115.2, + 113.0, + 115.2, + 115.2 + ], + "ttft_s": [ + 0.16, + 0.1, + 0.16, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9f0cfbcef69a73bf24c3254a60473fbf23d095540107ffbc5bb1d0300ebbec6a" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "3075eca197d4bdb7803275b0e8b52ca7fece2e4f5690e96237c7de6b4daff334" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9f0cfbcef69a73bf24c3254a60473fbf23d095540107ffbc5bb1d0300ebbec6a" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9f0cfbcef69a73bf24c3254a60473fbf23d095540107ffbc5bb1d0300ebbec6a" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 146.1, + "per_stream_tps": 146.1, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 142.80525444328643, + "makespan_s": 1.7926511247642338, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 147.01517397578954, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.7852733340114355, + "end_to_end_tps": 143.39540905188932, + "timing_valid": true, + "per_stream_tps": [ + 147.0 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + } + ], + "equal_alone": [ + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 146.1351758795083, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.7926511247642338, + "end_to_end_tps": 142.80525444328643, + "timing_valid": true, + "per_stream_tps": [ + 146.1 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + } + ], + "equal_alone": [ + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 142.2086362536904, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.844742250163108, + "end_to_end_tps": 138.77277434143716, + "timing_valid": true, + "per_stream_tps": [ + 142.2 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + } + ], + "equal_alone": [ + null + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 439.2, + "per_stream_tps": 111.9, + "ttft_s_max": 0.16, + "timing_valid": true, + "end_to_end_tps": 422.7879250253934, + "makespan_s": 2.422018083743751, + "steady_tps": 450.6, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 432.67965065963, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.457506666891277, + "end_to_end_tps": 416.68249115895605, + "timing_valid": true, + "steady_tps": 444.3043523234479, + "per_stream_tps": [ + 111.1, + 111.1, + 108.9, + 111.1 + ], + "ttft_s": [ + 0.16, + 0.16, + 0.1, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "20a5ae9731d2d95ef27f1c65a4b5d43536dacc13abdf0cdcd1fb00e4a1e8ab8e" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 439.15798068790735, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.422018083743751, + "end_to_end_tps": 422.7879250253934, + "timing_valid": true, + "steady_tps": 450.6206777370169, + "per_stream_tps": [ + 110.6, + 112.6, + 112.6, + 112.6 + ], + "ttft_s": [ + 0.1, + 0.16, + 0.16, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "20a5ae9731d2d95ef27f1c65a4b5d43536dacc13abdf0cdcd1fb00e4a1e8ab8e" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 442.19728263942795, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.4103812500834465, + "end_to_end_tps": 424.82905970354255, + "timing_valid": true, + "steady_tps": 453.91011736464424, + "per_stream_tps": [ + 113.5, + 113.5, + 111.3, + 113.5 + ], + "ttft_s": [ + 0.16, + 0.16, + 0.1, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "20a5ae9731d2d95ef27f1c65a4b5d43536dacc13abdf0cdcd1fb00e4a1e8ab8e" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + } + ] + } + ] + }, + { + "family": "vl", + "bits": 4, + "leg": 3, + "engine": "standard", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 1.0762107078917325, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "025e7799abd6977ff0c3185fc32ee75fbd4ba3d404535565ec46fae4915b795f" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.097270667087287, + "output_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "request_body_sha256": "ed436d1be8120ed043052d1801596681eb49c393fba11ef39b12b6ee94487bc5" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "unverified if server omits token hashes; text digest is not substituted" + }, + "memory": { + "max_rss_bytes": 2107703296, + "peak_physical_footprint_bytes": 3283946040 + }, + "tag": "vl-4-3-standard", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 144.4, + "per_stream_tps": 144.4, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 141.13385802187082, + "makespan_s": 1.8138808333314955, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 144.44133119730222, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.8138808333314955, + "end_to_end_tps": 141.13385802187082, + "timing_valid": true, + "per_stream_tps": [ + 144.4 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + } + ], + "equal_alone": [ + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 139.4979355076867, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.8765962501056492, + "end_to_end_tps": 136.41719681875503, + "timing_valid": true, + "per_stream_tps": [ + 139.5 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + } + ], + "equal_alone": [ + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 145.61124147462075, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.8010139167308807, + "end_to_end_tps": 142.14215538360727, + "timing_valid": true, + "per_stream_tps": [ + 145.6 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + } + ], + "equal_alone": [ + null + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 442.9, + "per_stream_tps": 113.6, + "ttft_s_max": 0.16, + "timing_valid": true, + "end_to_end_tps": 426.1327281481129, + "makespan_s": 2.403007167391479, + "steady_tps": 452.9, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 442.59445114490137, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.404666167218238, + "end_to_end_tps": 425.83873552168865, + "timing_valid": true, + "steady_tps": 452.92334546654286, + "per_stream_tps": [ + 113.6, + 111.4, + 113.6, + 113.6 + ], + "ttft_s": [ + 0.16, + 0.1, + 0.16, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9f0cfbcef69a73bf24c3254a60473fbf23d095540107ffbc5bb1d0300ebbec6a" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "3075eca197d4bdb7803275b0e8b52ca7fece2e4f5690e96237c7de6b4daff334" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9f0cfbcef69a73bf24c3254a60473fbf23d095540107ffbc5bb1d0300ebbec6a" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9f0cfbcef69a73bf24c3254a60473fbf23d095540107ffbc5bb1d0300ebbec6a" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 445.57812863196386, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.3891619169153273, + "end_to_end_tps": 428.6021775041925, + "timing_valid": true, + "steady_tps": 455.6928100491608, + "per_stream_tps": [ + 114.3, + 114.3, + 114.3, + 112.2 + ], + "ttft_s": [ + 0.16, + 0.16, + 0.16, + 0.1 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9f0cfbcef69a73bf24c3254a60473fbf23d095540107ffbc5bb1d0300ebbec6a" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9f0cfbcef69a73bf24c3254a60473fbf23d095540107ffbc5bb1d0300ebbec6a" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9f0cfbcef69a73bf24c3254a60473fbf23d095540107ffbc5bb1d0300ebbec6a" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "3075eca197d4bdb7803275b0e8b52ca7fece2e4f5690e96237c7de6b4daff334" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 442.85886016320626, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.403007167391479, + "end_to_end_tps": 426.1327281481129, + "timing_valid": true, + "steady_tps": 452.78263409830294, + "per_stream_tps": [ + 111.5, + 113.6, + 113.6, + 113.6 + ], + "ttft_s": [ + 0.1, + 0.16, + 0.16, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "3075eca197d4bdb7803275b0e8b52ca7fece2e4f5690e96237c7de6b4daff334" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9f0cfbcef69a73bf24c3254a60473fbf23d095540107ffbc5bb1d0300ebbec6a" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9f0cfbcef69a73bf24c3254a60473fbf23d095540107ffbc5bb1d0300ebbec6a" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9f0cfbcef69a73bf24c3254a60473fbf23d095540107ffbc5bb1d0300ebbec6a" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 142.2, + "per_stream_tps": 142.2, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 138.9944272855163, + "makespan_s": 1.8418004591949284, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 142.21828928668387, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.8418004591949284, + "end_to_end_tps": 138.9944272855163, + "timing_valid": true, + "per_stream_tps": [ + 142.2 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + } + ], + "equal_alone": [ + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 137.1517054872842, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.9112333334051073, + "end_to_end_tps": 133.94492212204312, + "timing_valid": true, + "per_stream_tps": [ + 137.2 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + } + ], + "equal_alone": [ + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 144.9788048407925, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.8104737089015543, + "end_to_end_tps": 141.39945735821794, + "timing_valid": true, + "per_stream_tps": [ + 145.0 + ], + "ttft_s": [ + 0.05 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + } + ], + "equal_alone": [ + null + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 445.3, + "per_stream_tps": 113.5, + "ttft_s_max": 0.16, + "timing_valid": true, + "end_to_end_tps": 428.4620938131924, + "makespan_s": 2.389943042304367, + "steady_tps": 457.0, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 445.3158422193051, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.389943042304367, + "end_to_end_tps": 428.4620938131924, + "timing_valid": true, + "steady_tps": 457.0113917554472, + "per_stream_tps": [ + 114.3, + 114.3, + 114.3, + 112.1 + ], + "ttft_s": [ + 0.16, + 0.16, + 0.16, + 0.1 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "20a5ae9731d2d95ef27f1c65a4b5d43536dacc13abdf0cdcd1fb00e4a1e8ab8e" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 438.6344021467109, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.4264929578639567, + "end_to_end_tps": 422.008230718884, + "timing_valid": true, + "steady_tps": 450.87876334536503, + "per_stream_tps": [ + 112.7, + 110.5, + 112.7, + 112.7 + ], + "ttft_s": [ + 0.16, + 0.1, + 0.16, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "20a5ae9731d2d95ef27f1c65a4b5d43536dacc13abdf0cdcd1fb00e4a1e8ab8e" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 446.9232375053428, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.3820332498289645, + "end_to_end_tps": 429.8848473561507, + "timing_valid": true, + "steady_tps": 458.45096242685065, + "per_stream_tps": [ + 114.6, + 114.6, + 112.5, + 114.6 + ], + "ttft_s": [ + 0.16, + 0.16, + 0.1, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "20a5ae9731d2d95ef27f1c65a4b5d43536dacc13abdf0cdcd1fb00e4a1e8ab8e" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "32ab60c2dbf46e6d42577bda7d2857c37ab3b0d166087ca8488be9195bd27f38" + } + ], + "equal_alone": [ + null, + null, + null, + null + ] + } + ] + } + ] + }, + { + "family": "vl", + "bits": 4, + "leg": 4, + "engine": "combined", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 3.4118054159916937, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 17.304976750165224, + "output_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1836220416, + "peak_physical_footprint_bytes": 2173945944 + }, + "tag": "vl-4-4-combined", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 147.8, + "per_stream_tps": 147.8, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 141.76196835904213, + "makespan_s": 1.8058440000750124, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 146.97943921569524, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.8159247911535203, + "end_to_end_tps": 140.97500141367775, + "timing_valid": true, + "per_stream_tps": [ + 147.0 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 148.29139956591266, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.800668082665652, + "end_to_end_tps": 142.16945502861677, + "timing_valid": true, + "per_stream_tps": [ + 148.3 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 147.76522481338233, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.8058440000750124, + "end_to_end_tps": 141.76196835904213, + "timing_valid": true, + "per_stream_tps": [ + 147.8 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 375.2, + "per_stream_tps": 98.2, + "ttft_s_max": 0.33, + "timing_valid": true, + "end_to_end_tps": 365.68555861736303, + "makespan_s": 2.800219959113747, + "steady_tps": 410.2, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 375.24419045268496, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.800219959113747, + "end_to_end_tps": 365.68555861736303, + "timing_valid": true, + "steady_tps": 410.22442603102627, + "per_stream_tps": [ + 99.7, + 96.7, + 102.9, + 93.8 + ], + "ttft_s": [ + 0.24, + 0.16, + 0.32, + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 375.56490070400423, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.7966721248812973, + "end_to_end_tps": 366.14946417555575, + "timing_valid": true, + "steady_tps": 411.0262999429127, + "per_stream_tps": [ + 96.7, + 103.1, + 99.8, + 93.9 + ], + "ttft_s": [ + 0.16, + 0.32, + 0.24, + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 373.9656367688199, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.8118270421400666, + "end_to_end_tps": 364.1760267091816, + "timing_valid": true, + "steady_tps": 410.0461564054171, + "per_stream_tps": [ + 96.4, + 99.6, + 93.5, + 102.8 + ], + "ttft_s": [ + 0.17, + 0.25, + 0.08, + 0.33 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 145.1, + "per_stream_tps": 145.1, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 139.19277210019735, + "makespan_s": 1.839175958186388, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 146.45046079001816, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.8228463749401271, + "end_to_end_tps": 140.4397010737718, + "timing_valid": true, + "per_stream_tps": [ + 146.5 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 145.11862556430796, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.839175958186388, + "end_to_end_tps": 139.19277210019735, + "timing_valid": true, + "per_stream_tps": [ + 145.1 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 143.40253101402254, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.860188249964267, + "end_to_end_tps": 137.62048008039918, + "timing_valid": true, + "per_stream_tps": [ + 143.4 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 376.2, + "per_stream_tps": 98.4, + "ttft_s_max": 0.33, + "timing_valid": true, + "end_to_end_tps": 366.5930015270739, + "makespan_s": 2.7932884581387043, + "steady_tps": 412.3, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 375.577824991286, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.798320250120014, + "end_to_end_tps": 365.9338133139275, + "timing_valid": true, + "steady_tps": 411.4732003495591, + "per_stream_tps": [ + 96.8, + 93.9, + 99.8, + 103.1 + ], + "ttft_s": [ + 0.16, + 0.08, + 0.24, + 0.32 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 376.65830631212634, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.789901874959469, + "end_to_end_tps": 367.03799842956, + "timing_valid": true, + "steady_tps": 412.9306448497089, + "per_stream_tps": [ + 103.5, + 100.2, + 97.1, + 94.2 + ], + "ttft_s": [ + 0.33, + 0.24, + 0.16, + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 376.2284539692673, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.7932884581387043, + "end_to_end_tps": 366.5930015270739, + "timing_valid": true, + "steady_tps": 412.3090187451913, + "per_stream_tps": [ + 94.1, + 100.0, + 103.3, + 96.9 + ], + "ttft_s": [ + 0.08, + 0.24, + 0.32, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + } + ] + } + ] + }, + { + "family": "vl", + "bits": 4, + "leg": 5, + "engine": "public", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 3.1815165420994163, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 13.796967291738838, + "output_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1836122112, + "peak_physical_footprint_bytes": 2173814944 + }, + "tag": "vl-4-5-public", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 154.0, + "per_stream_tps": 154.0, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 147.49705134190296, + "makespan_s": 1.7356279171071947, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 153.99289231927293, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.7356279171071947, + "end_to_end_tps": 147.49705134190296, + "timing_valid": true, + "per_stream_tps": [ + 154.0 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 155.53860784770828, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.719733249861747, + "end_to_end_tps": 148.86029564211796, + "timing_valid": true, + "per_stream_tps": [ + 155.5 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 152.32002433166892, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.7540543749928474, + "end_to_end_tps": 145.94758500633364, + "timing_valid": true, + "per_stream_tps": [ + 152.3 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 384.6, + "per_stream_tps": 100.2, + "ttft_s_max": 0.32, + "timing_valid": true, + "end_to_end_tps": 374.3003685716666, + "makespan_s": 2.735770749859512, + "steady_tps": 421.5, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 376.5159090924768, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.790206667035818, + "end_to_end_tps": 366.99790452721146, + "timing_valid": true, + "steady_tps": 411.9706113199974, + "per_stream_tps": [ + 97.0, + 100.0, + 103.3, + 94.1 + ], + "ttft_s": [ + 0.16, + 0.24, + 0.32, + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 384.55054971995594, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.735770749859512, + "end_to_end_tps": 374.3003685716666, + "timing_valid": true, + "steady_tps": 421.5293594229542, + "per_stream_tps": [ + 102.3, + 105.7, + 96.2, + 99.2 + ], + "ttft_s": [ + 0.24, + 0.32, + 0.08, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 389.2501387659456, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.7010359996929765, + "end_to_end_tps": 379.1137919362781, + "timing_valid": true, + "steady_tps": 426.88256002529783, + "per_stream_tps": [ + 103.6, + 107.0, + 97.3, + 100.4 + ], + "ttft_s": [ + 0.24, + 0.32, + 0.08, + 0.16 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 153.2, + "per_stream_tps": 153.2, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 146.47201622163075, + "makespan_s": 1.7477741250768304, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 153.19800841053, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.7477741250768304, + "end_to_end_tps": 146.47201622163075, + "timing_valid": true, + "per_stream_tps": [ + 153.2 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 154.44109753892025, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.7352849999442697, + "end_to_end_tps": 147.52619887120656, + "timing_valid": true, + "per_stream_tps": [ + 154.4 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 151.8266833789518, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 1.7603899580426514, + "end_to_end_tps": 145.42232465620413, + "timing_valid": true, + "per_stream_tps": [ + 151.8 + ], + "ttft_s": [ + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 382.5, + "per_stream_tps": 100.5, + "ttft_s_max": 0.33, + "timing_valid": true, + "end_to_end_tps": 372.584375687391, + "makespan_s": 2.748370749875903, + "steady_tps": 419.7, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 382.49961736388366, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.7486295830458403, + "end_to_end_tps": 372.5492901321663, + "timing_valid": true, + "steady_tps": 419.535454058573, + "per_stream_tps": [ + 105.1, + 98.6, + 101.7, + 95.6 + ], + "ttft_s": [ + 0.32, + 0.16, + 0.24, + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 384.93703635917745, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.7320836251601577, + "end_to_end_tps": 374.8055112844403, + "timing_valid": true, + "steady_tps": 423.4431661371578, + "per_stream_tps": [ + 106.1, + 96.3, + 99.3, + 102.7 + ], + "ttft_s": [ + 0.33, + 0.08, + 0.16, + 0.25 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 382.496204931923, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 2.748370749875903, + "end_to_end_tps": 372.584375687391, + "timing_valid": true, + "steady_tps": 419.7410992579244, + "per_stream_tps": [ + 98.6, + 101.8, + 105.2, + 95.6 + ], + "ttft_s": [ + 0.16, + 0.24, + 0.32, + 0.08 + ], + "errors": [], + "evidence": [ + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + } + ], + "equal_alone": [ + true, + true, + true, + true + ] + } + ] + } + ] + } + ], + "six_rep_medians": [ + { + "family": "text", + "engine": "public", + "prompt": "long-code-chat", + "streams": 1, + "existing_cell_tps": [ + 175.0, + 175.1 + ], + "six_existing_rep_tps": [ + 174.33357930951772, + 176.54645032063524, + 175.0280052102641, + 175.0803803635659, + 174.57396389141493, + 176.4727407119902 + ], + "median_six_rep_tps": 175.054192786915 + }, + { + "family": "text", + "engine": "public", + "prompt": "long-code-chat", + "streams": 4, + "existing_cell_tps": [ + 388.3, + 389.8 + ], + "six_existing_rep_tps": [ + 388.32442956935364, + 387.38800732228174, + 390.4867598842693, + 385.24300236921965, + 390.4341652635153, + 389.79263659668885 + ], + "median_six_rep_tps": 389.05853308302125 + }, + { + "family": "text", + "engine": "public", + "prompt": "long-prose-chat", + "streams": 1, + "existing_cell_tps": [ + 183.7, + 156.7 + ], + "six_existing_rep_tps": [ + 183.7156213823559, + 183.29057256770892, + 184.09310995785592, + 143.38614873922572, + 156.69145080631193, + 176.18895002172795 + ], + "median_six_rep_tps": 179.73976129471845 + }, + { + "family": "text", + "engine": "public", + "prompt": "long-prose-chat", + "streams": 4, + "existing_cell_tps": [ + 393.1, + 333.2 + ], + "six_existing_rep_tps": [ + 392.42369639073456, + 393.0500466804608, + 395.57580143014764, + 333.2213611523689, + 332.85472325385604, + 338.5504679373183 + ], + "median_six_rep_tps": 365.48708216402645 + }, + { + "family": "text", + "engine": "combined", + "prompt": "long-code-chat", + "streams": 1, + "existing_cell_tps": [ + 166.1, + 169.4 + ], + "six_existing_rep_tps": [ + 166.0869678921436, + 166.11623932730092, + 172.19134049652038, + 168.65410283796538, + 169.3786485536357, + 169.78297962149645 + ], + "median_six_rep_tps": 169.01637569580055 + }, + { + "family": "text", + "engine": "combined", + "prompt": "long-code-chat", + "streams": 4, + "existing_cell_tps": [ + 383.5, + 380.4 + ], + "six_existing_rep_tps": [ + 382.07900870977994, + 383.5114695784525, + 384.30602814425293, + 381.6083183952793, + 380.37264712928885, + 374.9253882115582 + ], + "median_six_rep_tps": 381.8436635525296 + }, + { + "family": "text", + "engine": "combined", + "prompt": "long-prose-chat", + "streams": 1, + "existing_cell_tps": [ + 177.9, + 175.0 + ], + "six_existing_rep_tps": [ + 177.79868486495323, + 178.15170226681155, + 177.90131392071672, + 174.98129958626902, + 175.40580218595554, + 174.17910834276157 + ], + "median_six_rep_tps": 176.60224352545438 + }, + { + "family": "text", + "engine": "combined", + "prompt": "long-prose-chat", + "streams": 4, + "existing_cell_tps": [ + 386.5, + 382.1 + ], + "six_existing_rep_tps": [ + 386.51572926730825, + 385.85576947303315, + 386.5338735839194, + 382.5669552494983, + 382.1196774371716, + 381.266970720118 + ], + "median_six_rep_tps": 384.21136236126574 + }, + { + "family": "text", + "engine": "standard", + "prompt": "long-code-chat", + "streams": 1, + "existing_cell_tps": [ + 169.8, + 172.4 + ], + "six_existing_rep_tps": [ + 172.26463880154267, + 168.81678185058905, + 169.78287602547306, + 174.06198297170022, + 172.36259324598905, + 171.4222667734278 + ], + "median_six_rep_tps": 171.84345278748523 + }, + { + "family": "text", + "engine": "standard", + "prompt": "long-code-chat", + "streams": 4, + "existing_cell_tps": [ + 429.2, + 472.0 + ], + "six_existing_rep_tps": [ + 446.02964748705443, + 429.1684099599845, + 361.63599877811157, + 472.1309561767671, + 471.99612113868454, + 469.87623090402326 + ], + "median_six_rep_tps": 457.9529391955389 + }, + { + "family": "text", + "engine": "standard", + "prompt": "long-prose-chat", + "streams": 1, + "existing_cell_tps": [ + 171.4, + 173.1 + ], + "six_existing_rep_tps": [ + 171.61466519462007, + 171.44383317187626, + 170.28580590814167, + 173.0794732633171, + 173.45309958433361, + 172.69256971026414 + ], + "median_six_rep_tps": 172.15361745244212 + }, + { + "family": "text", + "engine": "standard", + "prompt": "long-prose-chat", + "streams": 4, + "existing_cell_tps": [ + 483.7, + 473.6 + ], + "six_existing_rep_tps": [ + 484.0013914306858, + 483.69557655279243, + 472.22991978123383, + 473.58003752344314, + 451.4560536062305, + 479.2751294144382 + ], + "median_six_rep_tps": 476.42758346894067 + }, + { + "family": "vl", + "engine": "public", + "prompt": "long-code-chat", + "streams": 1, + "existing_cell_tps": [ + 152.1, + 154.0 + ], + "six_existing_rep_tps": [ + 152.05740745566536, + 152.6768289446356, + 152.10096918029834, + 153.99289231927293, + 155.53860784770828, + 152.32002433166892 + ], + "median_six_rep_tps": 152.49842663815224 + }, + { + "family": "vl", + "engine": "public", + "prompt": "long-code-chat", + "streams": 4, + "existing_cell_tps": [ + 379.8, + 384.6 + ], + "six_existing_rep_tps": [ + 382.4665220808524, + 379.75781631994755, + 343.2818701092185, + 376.5159090924768, + 384.55054971995594, + 389.2501387659456 + ], + "median_six_rep_tps": 381.1121692004 + }, + { + "family": "vl", + "engine": "public", + "prompt": "long-prose-chat", + "streams": 1, + "existing_cell_tps": [ + 150.5, + 153.2 + ], + "six_existing_rep_tps": [ + 148.1346901549664, + 150.4833021557889, + 152.05146118191536, + 153.19800841053, + 154.44109753892025, + 151.8266833789518 + ], + "median_six_rep_tps": 151.93907228043358 + }, + { + "family": "vl", + "engine": "public", + "prompt": "long-prose-chat", + "streams": 4, + "existing_cell_tps": [ + 382.6, + 382.5 + ], + "six_existing_rep_tps": [ + 384.2723782007661, + 382.5557575031354, + 379.8426799933923, + 382.49961736388366, + 384.93703635917745, + 382.496204931923 + ], + "median_six_rep_tps": 382.5276874335095 + }, + { + "family": "vl", + "engine": "combined", + "prompt": "long-code-chat", + "streams": 1, + "existing_cell_tps": [ + 146.2, + 147.8 + ], + "six_existing_rep_tps": [ + 146.20333234391993, + 146.5534788207802, + 139.5312984437731, + 146.97943921569524, + 148.29139956591266, + 147.76522481338233 + ], + "median_six_rep_tps": 146.76645901823773 + }, + { + "family": "vl", + "engine": "combined", + "prompt": "long-code-chat", + "streams": 4, + "existing_cell_tps": [ + 372.0, + 375.2 + ], + "six_existing_rep_tps": [ + 373.2887956221813, + 371.9627317535548, + 371.86516750959413, + 375.24419045268496, + 375.56490070400423, + 373.9656367688199 + ], + "median_six_rep_tps": 373.6272161955006 + }, + { + "family": "vl", + "engine": "combined", + "prompt": "long-prose-chat", + "streams": 1, + "existing_cell_tps": [ + 146.1, + 145.1 + ], + "six_existing_rep_tps": [ + 146.13235292633067, + 146.63771935020262, + 145.76576185599458, + 146.45046079001816, + 145.11862556430796, + 143.40253101402254 + ], + "median_six_rep_tps": 145.94905739116263 + }, + { + "family": "vl", + "engine": "combined", + "prompt": "long-prose-chat", + "streams": 4, + "existing_cell_tps": [ + 372.0, + 376.2 + ], + "six_existing_rep_tps": [ + 374.5021496466539, + 371.0655850940012, + 371.9541637702016, + 375.577824991286, + 376.65830631212634, + 376.2284539692673 + ], + "median_six_rep_tps": 375.0399873189699 + }, + { + "family": "vl", + "engine": "standard", + "prompt": "long-code-chat", + "streams": 1, + "existing_cell_tps": [ + 145.4, + 144.4 + ], + "six_existing_rep_tps": [ + 145.39878073480114, + 145.48893022521824, + 141.39606992674067, + 144.44133119730222, + 139.4979355076867, + 145.61124147462075 + ], + "median_six_rep_tps": 144.92005596605168 + }, + { + "family": "vl", + "engine": "standard", + "prompt": "long-code-chat", + "streams": 4, + "existing_cell_tps": [ + 436.3, + 442.9 + ], + "six_existing_rep_tps": [ + 425.2857757503177, + 436.27337210105026, + 448.9789523933043, + 442.59445114490137, + 445.57812863196386, + 442.85886016320626 + ], + "median_six_rep_tps": 442.72665565405384 + }, + { + "family": "vl", + "engine": "standard", + "prompt": "long-prose-chat", + "streams": 1, + "existing_cell_tps": [ + 146.1, + 142.2 + ], + "six_existing_rep_tps": [ + 147.01517397578954, + 146.1351758795083, + 142.2086362536904, + 142.21828928668387, + 137.1517054872842, + 144.9788048407925 + ], + "median_six_rep_tps": 143.59854706373818 + }, + { + "family": "vl", + "engine": "standard", + "prompt": "long-prose-chat", + "streams": 4, + "existing_cell_tps": [ + 439.2, + 445.3 + ], + "six_existing_rep_tps": [ + 432.67965065963, + 439.15798068790735, + 442.19728263942795, + 445.3158422193051, + 438.6344021467109, + 446.9232375053428 + ], + "median_six_rep_tps": 440.6776316636676 + } + ], + "cold": [ + { + "family": "text", + "engine": "public", + "prompt_tokens": 2048, + "ttft_s": [ + 3.1976742083206773, + 3.1604983750730753 + ], + "output_sha256": [ + "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9" + ] + }, + { + "family": "text", + "engine": "public", + "prompt_tokens": 8192, + "ttft_s": [ + 13.849388832692057, + 14.409123916644603 + ], + "output_sha256": [ + "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349" + ] + }, + { + "family": "text", + "engine": "combined", + "prompt_tokens": 2048, + "ttft_s": [ + 3.4472638331353664, + 3.3981029582209885 + ], + "output_sha256": [ + "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9" + ] + }, + { + "family": "text", + "engine": "combined", + "prompt_tokens": 8192, + "ttft_s": [ + 17.29660391714424, + 17.221552750095725 + ], + "output_sha256": [ + "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349" + ] + }, + { + "family": "text", + "engine": "standard", + "prompt_tokens": 2048, + "ttft_s": [ + 1.4001411660574377, + 1.0871214168146253 + ], + "output_sha256": [ + "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9" + ] + }, + { + "family": "text", + "engine": "standard", + "prompt_tokens": 8192, + "ttft_s": [ + 4.058180375024676, + 4.053703292272985 + ], + "output_sha256": [ + "3855897e8f18058e244110a049fbe9ab9131d137fd6b116ec4d426c83093a1b9", + "3855897e8f18058e244110a049fbe9ab9131d137fd6b116ec4d426c83093a1b9" + ] + }, + { + "family": "vl", + "engine": "public", + "prompt_tokens": 2048, + "ttft_s": [ + 4.068471458274871, + 3.1815165420994163 + ], + "output_sha256": [ + "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e" + ] + }, + { + "family": "vl", + "engine": "public", + "prompt_tokens": 8192, + "ttft_s": [ + 14.948924167081714, + 13.796967291738838 + ], + "output_sha256": [ + "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e" + ] + }, + { + "family": "vl", + "engine": "combined", + "prompt_tokens": 2048, + "ttft_s": [ + 3.4114462919533253, + 3.4118054159916937 + ], + "output_sha256": [ + "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e" + ] + }, + { + "family": "vl", + "engine": "combined", + "prompt_tokens": 8192, + "ttft_s": [ + 17.34155425010249, + 17.304976750165224 + ], + "output_sha256": [ + "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e" + ] + }, + { + "family": "vl", + "engine": "standard", + "prompt_tokens": 2048, + "ttft_s": [ + 1.0534550421871245, + 1.0762107078917325 + ], + "output_sha256": [ + "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e" + ] + }, + { + "family": "vl", + "engine": "standard", + "prompt_tokens": 8192, + "ttft_s": [ + 4.1006536250934005, + 4.097270667087287 + ], + "output_sha256": [ + "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e" + ] + } + ], + "physical_footprint": [ + { + "family": "text", + "engine": "public", + "legs": [ + { + "max_rss_bytes": 1217773568, + "peak_physical_footprint_bytes": 1514243752 + }, + { + "max_rss_bytes": 1217822720, + "peak_physical_footprint_bytes": 1514260112 + } + ] + }, + { + "family": "text", + "engine": "combined", + "legs": [ + { + "max_rss_bytes": 1218560000, + "peak_physical_footprint_bytes": 1514964696 + }, + { + "max_rss_bytes": 1217888256, + "peak_physical_footprint_bytes": 1514325720 + } + ] + }, + { + "family": "text", + "engine": "standard", + "legs": [ + { + "max_rss_bytes": 1509490688, + "peak_physical_footprint_bytes": 2763966528 + }, + { + "max_rss_bytes": 1487306752, + "peak_physical_footprint_bytes": 2758903896 + } + ] + }, + { + "family": "vl", + "engine": "public", + "legs": [ + { + "max_rss_bytes": 1835630592, + "peak_physical_footprint_bytes": 2173503600 + }, + { + "max_rss_bytes": 1836122112, + "peak_physical_footprint_bytes": 2173814944 + } + ] + }, + { + "family": "vl", + "engine": "combined", + "legs": [ + { + "max_rss_bytes": 1836417024, + "peak_physical_footprint_bytes": 2174273720 + }, + { + "max_rss_bytes": 1836220416, + "peak_physical_footprint_bytes": 2173945944 + } + ] + }, + { + "family": "vl", + "engine": "standard", + "legs": [ + { + "max_rss_bytes": 2108932096, + "peak_physical_footprint_bytes": 3283257768 + }, + { + "max_rss_bytes": 2107703296, + "peak_physical_footprint_bytes": 3283946040 + } + ] + } + ] + }, + "limitations": [ + "Standard omits token hashes: alone/serial token equality unverified; output text digests are not substituted.", + "Both engines use request budget70000, but native reserved cache and standard lazy KV/sliding cache allocation differ.", + "Frozen public21728/ad3 baseline versus qualified combinedad72; later public builds are not substituted.", + "No6/8bit or32k/64k measurements in this phase.", + "Code/prose request workload for VL model is text-only; image correctness is separate qualified evidence.", + "Public text prose reverse leg drifted from183.7 to156.7solo and393.1 to333.2four-stream; retain per-leg spread. No thermal, frequency or bandwidth counter attribution.", + "The combined binary is the frozen qualified branchless+postnorm candidate; later applied-public binaries are not substituted.", + "Standard request body model IDs differ from native aliases, so request-body hashes differ even for the same pinned message fixture.", + "Standard8k text cold generated a different output digest from native; timing evidence is not a whole-model fidelity proof.", + "Peak physical footprint is the actual /usr/bin/time -l process measurement, not a claimed GPU allocator or pinned-memory peak." + ], + "raw_file_manifest": { + "events.json": { + "bytes": 30478, + "sha256": "ddbc001617d38543feb37798395b11dd4ef682e6c546d5805888bbf743d6d65b" + }, + "independent-review.json": { + "bytes": 3624, + "sha256": "9e68ef3a90ea418cf844f1333244a5d46cda6a91c9d738d59178e001cd706a92" + }, + "independent-review.py": { + "bytes": 4080, + "sha256": "9f0282acde5693fa1442b1eab9dcbe29249555ca625e4b42fae880a5408a3420" + }, + "manifest.json": { + "bytes": 12788, + "sha256": "22148de349b615a3b8b72ef17f98e6623f202aa26c0766cf2e476f503f38ab71" + }, + "phase1-summary.json": { + "bytes": 17748, + "sha256": "ed46d838704efdcda3cd097072f088ecad013fd70ccd719f14026d33933caaeb" + }, + "records.json": { + "bytes": 12701, + "sha256": "ce8583512252c2ae87b57b77835530aab44aecea40e231a641a9ee67f4534a94" + }, + "text-4-0-public.after.health": { + "bytes": 298, + "sha256": "421a862e6e4c08711e03c75307d1ad907b9538c77e95145c995ff22ab1e52a98" + }, + "text-4-0-public.after.metrics": { + "bytes": 12299, + "sha256": "3505202fb9381f863cf3ddb40483968079867d175c10265d65bc3305b8116c9c" + }, + "text-4-0-public.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-0-public.before.health": { + "bytes": 296, + "sha256": "8c789a277ca189a0923767fa83532a352de4a11334a37855d824cd2137333df4" + }, + "text-4-0-public.before.metrics": { + "bytes": 12128, + "sha256": "f9da163dbf442d34b0fcb0d7abcf3d4d34ac59c4bbe624dd5d8c4181d856fffc" + }, + "text-4-0-public.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-0-public.cold-2048.client.log": { + "bytes": 348, + "sha256": "84d78b3202f5c276158c6314db0f0d53a1af25c5d3e8093bc38ceff524217c1e" + }, + "text-4-0-public.cold-2048.json": { + "bytes": 20294, + "sha256": "9a054d2222ea0b76f2fc008fb0d38707924d590ca0999d3048452e3a577198cc" + }, + "text-4-0-public.cold-8192.client.log": { + "bytes": 349, + "sha256": "aaec18c32ab79dfdd565b1052483fec760046d31c083bfba955203b96684c7b0" + }, + "text-4-0-public.cold-8192.json": { + "bytes": 57151, + "sha256": "277c3104bf3e7b4911edf6b8e7bd58a914f98d32a1a25d67506758748a63d03f" + }, + "text-4-0-public.concurrent.client.log": { + "bytes": 3232, + "sha256": "8261f032cb35002bba510cd84e96b03808c398ff870adb74a87eacf10b8f91f6" + }, + "text-4-0-public.concurrent.json": { + "bytes": 280772, + "sha256": "e1cfbb2f30e112b8d3cb1202d456abf4aaf94aed9f8362e0d0e37218f5843dda" + }, + "text-4-0-public.server.log": { + "bytes": 13609, + "sha256": "c9ab4413703b0d7e8f41c11c56f34e036832bacf8bdacc8454af0041c12887be" + }, + "text-4-1-combined.after.health": { + "bytes": 298, + "sha256": "9903e3b95f46b71d8186f3abe415a96ef62d0a94ceaa7f3efd7b1208a0cefd36" + }, + "text-4-1-combined.after.metrics": { + "bytes": 12303, + "sha256": "aeec875bc3c6effbda360d32501dbfc3376a2db4c8e58a4c0afb3785845860c3" + }, + "text-4-1-combined.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-1-combined.before.health": { + "bytes": 296, + "sha256": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881" + }, + "text-4-1-combined.before.metrics": { + "bytes": 12128, + "sha256": "cfd6e0d1abeb10f092e108c9090eb92ff5e62c04adf5a845e34ad44968fec4b3" + }, + "text-4-1-combined.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-1-combined.cold-2048.client.log": { + "bytes": 349, + "sha256": "97526b0326a0fcf617cb4a2cc77fff29599d75524b655e36db0c4121b132129e" + }, + "text-4-1-combined.cold-2048.json": { + "bytes": 20295, + "sha256": "1a889aaf0e6e69e26d777522a3365cdd6729154e300fbb0d89caefdb29ba96db" + }, + "text-4-1-combined.cold-8192.client.log": { + "bytes": 348, + "sha256": "88b8ac9358e175852c77bae8a353f676e663a46702ddf838b275e1ed44f99e45" + }, + "text-4-1-combined.cold-8192.json": { + "bytes": 57150, + "sha256": "06581ab3330b40c0bb2c3482400880f2fe9e68e32abaabb954c07c5e02100cd7" + }, + "text-4-1-combined.concurrent.client.log": { + "bytes": 3232, + "sha256": "1d71a2b74dc478fb756de4371090853d9d2ddd142cdd0adb3fc0dfef4a27d39b" + }, + "text-4-1-combined.concurrent.json": { + "bytes": 281297, + "sha256": "ac39d743350fd459e644f2cb1b8c547d5468ebdb2936ce00e29a2fcb5108f495" + }, + "text-4-1-combined.server.log": { + "bytes": 13603, + "sha256": "1edf3e0031171bc652ad5e0c477aa3151b124617a454d8df4ca91e5d36edc17b" + }, + "text-4-2-standard.after.health": { + "bytes": 449, + "sha256": "17e6666ff1419af1e0f5f493004feab65b09c9d7d8bfd22fbacc102fa8729e6c" + }, + "text-4-2-standard.after.metrics": { + "bytes": 31998, + "sha256": "7504f2b8de52f1e5cc7ce1308e3cca240bd073cbac4a28a20597369ec7c1128a" + }, + "text-4-2-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-4-2-standard.before.health": { + "bytes": 449, + "sha256": "17e6666ff1419af1e0f5f493004feab65b09c9d7d8bfd22fbacc102fa8729e6c" + }, + "text-4-2-standard.before.metrics": { + "bytes": 905, + "sha256": "26b51aaf2c7f37eb81e52b011882c77ed30331e6181ec9a2d924d68ea14c30a1" + }, + "text-4-2-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-4-2-standard.cold-2048.client.log": { + "bytes": 295, + "sha256": "5232bc69406a15fe6603220d1fab78c88288aa16d0e1115fb0d6a8994a3c8f6a" + }, + "text-4-2-standard.cold-2048.json": { + "bytes": 26045, + "sha256": "0d204373fa161d53573b2474654d35008670be3c7ccdaed45f254d19f8aa4f29" + }, + "text-4-2-standard.cold-8192.client.log": { + "bytes": 293, + "sha256": "eded448fa26377b41ac8ef22d7f9d02810b751ac2d21bab59b27ff9a6cbddd5e" + }, + "text-4-2-standard.cold-8192.json": { + "bytes": 62899, + "sha256": "75ba4fdded3c69ad4fb46ce77963ff70340245c645308f7f094fc0830185ad4e" + }, + "text-4-2-standard.concurrent.client.log": { + "bytes": 3238, + "sha256": "080ed7a5a6cb6ca212d6936d1746d94ca6927fb9b49d60b461639a8fa31f5223" + }, + "text-4-2-standard.concurrent.json": { + "bytes": 283021, + "sha256": "e8fa24b3febd6e2716963ad2981697b25b0f9a7fa6f9dddd23be5edbc6c82db0" + }, + "text-4-2-standard.server.log": { + "bytes": 50717, + "sha256": "b9a8190dbd56dc7af5c097ef265e70232ddfe46b3506164338cea5df330e713a" + }, + "text-4-3-standard.after.health": { + "bytes": 449, + "sha256": "17e6666ff1419af1e0f5f493004feab65b09c9d7d8bfd22fbacc102fa8729e6c" + }, + "text-4-3-standard.after.metrics": { + "bytes": 32022, + "sha256": "727d50eb840467c1181ecdb08a26f4263b7c33be916d9692b85d29809f2005ff" + }, + "text-4-3-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-4-3-standard.before.health": { + "bytes": 449, + "sha256": "17e6666ff1419af1e0f5f493004feab65b09c9d7d8bfd22fbacc102fa8729e6c" + }, + "text-4-3-standard.before.metrics": { + "bytes": 904, + "sha256": "52c02d19f03dd438ecfe12fa4c3d4ca92e37ae8de76520d008694f76872cc44d" + }, + "text-4-3-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-4-3-standard.cold-2048.client.log": { + "bytes": 294, + "sha256": "1a818bccb1f78515b6b86bcbd143a047071af7893927540b939d273455132137" + }, + "text-4-3-standard.cold-2048.json": { + "bytes": 26043, + "sha256": "18140744926db72ff2fd3eb2504d19087632a1a01bcf206c5cfddbd544676e02" + }, + "text-4-3-standard.cold-8192.client.log": { + "bytes": 293, + "sha256": "4d95a17179bf0f5f62ce4d042e055b8cc0483248cdb809f34093ab0c5b4aa87b" + }, + "text-4-3-standard.cold-8192.json": { + "bytes": 62907, + "sha256": "13d362f5cfdf2feca0d3a28e0144d52257efc37643fe165a2169d31a55c80380" + }, + "text-4-3-standard.concurrent.client.log": { + "bytes": 3238, + "sha256": "57f25bd1a3669b2772c8b4e6509ce3ee82b1b5a8de7bcc54555f727decbebe11" + }, + "text-4-3-standard.concurrent.json": { + "bytes": 283040, + "sha256": "ef6a2b046061524f59ae6615c7e14d5bce0a33ab51c91a50a9c6a23e53850156" + }, + "text-4-3-standard.server.log": { + "bytes": 50731, + "sha256": "c7dbece91bd53f5ed14b7efe779784a220093f59f8aa53fab624cdaf9cd03cae" + }, + "text-4-4-combined.after.health": { + "bytes": 298, + "sha256": "9903e3b95f46b71d8186f3abe415a96ef62d0a94ceaa7f3efd7b1208a0cefd36" + }, + "text-4-4-combined.after.metrics": { + "bytes": 12303, + "sha256": "4ee4130732a297bb67e316569691cd7a7b5eb04a8fe80a0a721ebf617cfa7fbc" + }, + "text-4-4-combined.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-4-combined.before.health": { + "bytes": 296, + "sha256": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881" + }, + "text-4-4-combined.before.metrics": { + "bytes": 12128, + "sha256": "b28b48959fb7e2e4e9d2047da1f093a37e48c658ab53ae37b960b4d827c18a89" + }, + "text-4-4-combined.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-4-combined.cold-2048.client.log": { + "bytes": 349, + "sha256": "40872192dae33ec46b14b07f73dc5a2dd90ea4091851523caa7ebd621869185f" + }, + "text-4-4-combined.cold-2048.json": { + "bytes": 20295, + "sha256": "e2650773b810b1fd00486daa5dcfa4a6bfb01cf882b372d9fc6e18031276519c" + }, + "text-4-4-combined.cold-8192.client.log": { + "bytes": 349, + "sha256": "97e0bad59aef3280d02619c611b88dd29933bff598e42f52b4428ffba81f3907" + }, + "text-4-4-combined.cold-8192.json": { + "bytes": 57150, + "sha256": "758bdf35f0f1065f4c37702493e1f36757d39a2502e27cbd34abc3117433ab35" + }, + "text-4-4-combined.concurrent.client.log": { + "bytes": 3239, + "sha256": "2174df350dde3de1704e9b3850ed72dd9c6ca3fde56f799542243c52c7260407" + }, + "text-4-4-combined.concurrent.json": { + "bytes": 280318, + "sha256": "6daa98247532da73a09c12c81806bd355ac8cc8ef884060b71bfd8c6d5b2df21" + }, + "text-4-4-combined.server.log": { + "bytes": 13602, + "sha256": "7e11c8a67d6d286c5d65d2605133d995a226613c7d119ad7ffc18f29e72cbd7d" + }, + "text-4-5-public.after.health": { + "bytes": 298, + "sha256": "82defd35723aee950c60a4c840c6b1adceebb8d1001e61eab8892f003e8b6467" + }, + "text-4-5-public.after.metrics": { + "bytes": 12301, + "sha256": "872a11da70e6d602be6d87b90c91f3f97cb82da071bd55bbdcf49802416e8c2d" + }, + "text-4-5-public.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-5-public.before.health": { + "bytes": 296, + "sha256": "8c789a277ca189a0923767fa83532a352de4a11334a37855d824cd2137333df4" + }, + "text-4-5-public.before.metrics": { + "bytes": 12128, + "sha256": "5f6bdd7fa09ed697c0b795006457190705731c37d83f15a2c0dfca013b3f1879" + }, + "text-4-5-public.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-4-5-public.cold-2048.client.log": { + "bytes": 348, + "sha256": "51d9fd137de6a4c0ffab39077bfd9d6a7fc9bb81ccdafeafb1bba03357b41931" + }, + "text-4-5-public.cold-2048.json": { + "bytes": 20293, + "sha256": "ac543c8047e3b0943f0c74d4eb3a6efcb967c74396a1574e682fa46fb57d3080" + }, + "text-4-5-public.cold-8192.client.log": { + "bytes": 349, + "sha256": "ff21237fe43ac9c4c13dc90a310a74400671fc236eca825c2138591893475f8b" + }, + "text-4-5-public.cold-8192.json": { + "bytes": 57151, + "sha256": "9e83eb99eedf33e4704193ff6594938c62398a19e39f93f13630fc2444d46172" + }, + "text-4-5-public.concurrent.client.log": { + "bytes": 3235, + "sha256": "5053ac4e028b6b3f2af11be70c86168d603286c6ac8598024def19e4c94d3243" + }, + "text-4-5-public.concurrent.json": { + "bytes": 280185, + "sha256": "3410745c9d258cb5b6505f23addef85b3b28a9ed7e0e3db673dc1b466c2fdda3" + }, + "text-4-5-public.server.log": { + "bytes": 13599, + "sha256": "685a364b7fe651a2b3410f6935d3af4c6ab712b60323766ff4a2c7598908fd48" + }, + "vl-4-0-public.after.health": { + "bytes": 298, + "sha256": "09f638fe46665554902ba68e5e5d2aab248272c8127d38439fc3a532615ee5a5" + }, + "vl-4-0-public.after.metrics": { + "bytes": 12303, + "sha256": "de2917cfeb4470e9304a486309c3f4c7f4043c1d58b1f52a2e31dfc68e2381bc" + }, + "vl-4-0-public.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-0-public.before.health": { + "bytes": 296, + "sha256": "8c789a277ca189a0923767fa83532a352de4a11334a37855d824cd2137333df4" + }, + "vl-4-0-public.before.metrics": { + "bytes": 12128, + "sha256": "79b1c2a5ef34b5dd904fba6de209971719abbf3c707b15890c7f71f8ed9b10ed" + }, + "vl-4-0-public.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-0-public.cold-2048.client.log": { + "bytes": 347, + "sha256": "d35590d9f126609b741c99fe8ee4d0e52956d92a931fa51353d4e692394be02d" + }, + "vl-4-0-public.cold-2048.json": { + "bytes": 20276, + "sha256": "d944cfcb4f29cc2ea6f2a63ff8a643655c6df80f9b985e346135b7061af48448" + }, + "vl-4-0-public.cold-8192.client.log": { + "bytes": 349, + "sha256": "6e806eb7422f152005c179f5b8d331726e90e697f00aa86df94c3fbe68d7e5f2" + }, + "vl-4-0-public.cold-8192.json": { + "bytes": 57149, + "sha256": "38166978d5bc8b52b966f94e288f8eb1d362b3dff7f91be59f723b820ed1bbea" + }, + "vl-4-0-public.concurrent.client.log": { + "bytes": 3226, + "sha256": "08cd1fa665e4ee03b932ac013de823c2b0b3df12e3a237a5622655d76977ff42" + }, + "vl-4-0-public.concurrent.json": { + "bytes": 287529, + "sha256": "052ebbc993b194428d9027ab5a6c39d603f1b4739d1e0d611b206ab1bcdc254f" + }, + "vl-4-0-public.server.log": { + "bytes": 13573, + "sha256": "de45910ec4438e5d824343cac9a14e84604777b396e7ac73a302a7a5ae1dbd31" + }, + "vl-4-1-combined.after.health": { + "bytes": 298, + "sha256": "fb2dc251074dfd4ba9f6b848f9f3528656ae1a5a4d3971bd0c589da581e658b9" + }, + "vl-4-1-combined.after.metrics": { + "bytes": 12303, + "sha256": "5e189b70364a948aa2735cf382a48986244b849f88c068732bdca771554fd091" + }, + "vl-4-1-combined.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-1-combined.before.health": { + "bytes": 296, + "sha256": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881" + }, + "vl-4-1-combined.before.metrics": { + "bytes": 12128, + "sha256": "d8eba96d174b50600e16310f453c1ae9f0c7162952a9c454db18f5de4ca78d5e" + }, + "vl-4-1-combined.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-1-combined.cold-2048.client.log": { + "bytes": 348, + "sha256": "252af0af9a08eece3d674e80f6524935ca54e858472c31978be732baf7d21e71" + }, + "vl-4-1-combined.cold-2048.json": { + "bytes": 20278, + "sha256": "e7942deea75684f0b048bca65b214a2c9ae152e920968197d14a5ac141895f88" + }, + "vl-4-1-combined.cold-8192.client.log": { + "bytes": 348, + "sha256": "689bd543a30adce4a207841e3ee387798ff63a72c47f74fe6102e339864ee1c1" + }, + "vl-4-1-combined.cold-8192.json": { + "bytes": 57148, + "sha256": "870f7e29aaf89881981dc6fe9e0fb9986d18eeb43bf15e22086e957da6e31fc5" + }, + "vl-4-1-combined.concurrent.client.log": { + "bytes": 3239, + "sha256": "9b4ba7b70a672e1f1f85f158baf525fed1cedd44fd51b68cc32f375483d0fb25" + }, + "vl-4-1-combined.concurrent.json": { + "bytes": 286751, + "sha256": "3efc679a0fcef439bef81a74e1fe750ad5be379900c65f58b4faca6a5b3b7b4b" + }, + "vl-4-1-combined.server.log": { + "bytes": 13569, + "sha256": "b82736585b367f1649769ab2100226a0e1f41a32bbf8570846f5f4dcd0b94a3c" + }, + "vl-4-2-standard.after.health": { + "bytes": 453, + "sha256": "ee45e784aa99341cf066d20507920a28dfdcd5fde13e24adccd716753aa53736" + }, + "vl-4-2-standard.after.metrics": { + "bytes": 32069, + "sha256": "fc1c234db5b1f36962a84a45b04d43d3ae8ae3b7950c0143e6d4788a32922f4c" + }, + "vl-4-2-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-4-2-standard.before.health": { + "bytes": 453, + "sha256": "ee45e784aa99341cf066d20507920a28dfdcd5fde13e24adccd716753aa53736" + }, + "vl-4-2-standard.before.metrics": { + "bytes": 909, + "sha256": "e5a24d4c419a5736bede8ea24c4fba04fcf40ec17f8f404582f65af0cd0dc5d3" + }, + "vl-4-2-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-4-2-standard.cold-2048.client.log": { + "bytes": 295, + "sha256": "54d5c9ee5dd3d043062cce61e44b34cfbb5565f91bfcba2605cda76ce3f5df4b" + }, + "vl-4-2-standard.cold-2048.json": { + "bytes": 26043, + "sha256": "6af7c1da5abf82bcc2fed9bca5d8e9dd638b0b1e06c29a0a208863bb7e46a4ba" + }, + "vl-4-2-standard.cold-8192.client.log": { + "bytes": 294, + "sha256": "e44a9c8b1f48e893522b7890e131d5494befd48da34d6f71c6c4695497871e67" + }, + "vl-4-2-standard.cold-8192.json": { + "bytes": 62916, + "sha256": "67b1b0ad360abae12049d8690d6a46aca76104f08f95aa919c32b47055655fc2" + }, + "vl-4-2-standard.concurrent.client.log": { + "bytes": 3216, + "sha256": "abdbcc7a15c3eb186fc85bc76fa498801472a8b9c2db28d396ea8962b00a3e7e" + }, + "vl-4-2-standard.concurrent.json": { + "bytes": 283051, + "sha256": "ac20c8633194c6cd981758a8198175a31fdbd93c8194fa13c15d1f8e4130e68d" + }, + "vl-4-2-standard.server.log": { + "bytes": 50797, + "sha256": "ce86d937b89953711b96a00f73930805a0d9b6919e97bbf8ee8239c4a92561b7" + }, + "vl-4-3-standard.after.health": { + "bytes": 453, + "sha256": "ee45e784aa99341cf066d20507920a28dfdcd5fde13e24adccd716753aa53736" + }, + "vl-4-3-standard.after.metrics": { + "bytes": 32011, + "sha256": "521bd11ae4eae455a5d6cfc07a508fd1e1dcc090adf9d650777c7e65548f01b8" + }, + "vl-4-3-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-4-3-standard.before.health": { + "bytes": 453, + "sha256": "ee45e784aa99341cf066d20507920a28dfdcd5fde13e24adccd716753aa53736" + }, + "vl-4-3-standard.before.metrics": { + "bytes": 909, + "sha256": "e1ec519ebc5d5824395fadc332516be06266a6cb81de8b1c6739e1c3a848a6a9" + }, + "vl-4-3-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-4-3-standard.cold-2048.client.log": { + "bytes": 295, + "sha256": "dcdc8c4ab5a72f64c566af4af21a88e814cbee69eb53fc36e9b552f81e183974" + }, + "vl-4-3-standard.cold-2048.json": { + "bytes": 26046, + "sha256": "c44814cf4c67055eebe95f0e5bf15bc58886810ece4e1ae4b7626f93bf6fa40d" + }, + "vl-4-3-standard.cold-8192.client.log": { + "bytes": 293, + "sha256": "a0974b13161bfdb36627d6629646d57f8dde103846682f98ebb828278ab6c15a" + }, + "vl-4-3-standard.cold-8192.json": { + "bytes": 62911, + "sha256": "691aa208bd7c11ca53b0f6daa2dbee882900716daf48d57a167e30669291cf26" + }, + "vl-4-3-standard.concurrent.client.log": { + "bytes": 3212, + "sha256": "ddc272d953ace7882a258fda730de7b6ce21d559c1ebb188beffd889cc35838c" + }, + "vl-4-3-standard.concurrent.json": { + "bytes": 283078, + "sha256": "1e7609728da420a901fb21738af418aa5254bea9855efab6e06c158aefdb259a" + }, + "vl-4-3-standard.server.log": { + "bytes": 50803, + "sha256": "2a958023dc60e1536754f850d6b645ac6ed193710511ac5252efe9b557810365" + }, + "vl-4-4-combined.after.health": { + "bytes": 298, + "sha256": "9903e3b95f46b71d8186f3abe415a96ef62d0a94ceaa7f3efd7b1208a0cefd36" + }, + "vl-4-4-combined.after.metrics": { + "bytes": 12303, + "sha256": "410a205658fbdf201e0613fbe59d7fac62308a4631ee871749d1af92df31feae" + }, + "vl-4-4-combined.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-4-combined.before.health": { + "bytes": 296, + "sha256": "3ecca835aef6168623e5fd09af1a553676471c6c3d76176c12322725ac4e2881" + }, + "vl-4-4-combined.before.metrics": { + "bytes": 12128, + "sha256": "f13f0e533029c9905c333fab10297ef89759d5e0531ea3d66d6c2b1a82a78c44" + }, + "vl-4-4-combined.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-4-combined.cold-2048.client.log": { + "bytes": 349, + "sha256": "511f761242e9bfd82f6052009f4b676cc1e516bbfd14f77111fe6ce6daa46439" + }, + "vl-4-4-combined.cold-2048.json": { + "bytes": 20278, + "sha256": "e2e455d5bf249cfa628a8b9301ed667f5d114b7ec3c05f34b04946a374d309ba" + }, + "vl-4-4-combined.cold-8192.client.log": { + "bytes": 348, + "sha256": "8030fa144d76c7d314242dcdfc70dd804854c9ef358d8f1d8ff283563ce4296c" + }, + "vl-4-4-combined.cold-8192.json": { + "bytes": 57148, + "sha256": "872ee644c6d57e1b4dcef93c8dd7fcb1cb68647a4a8494385b5d0d8fc311c031" + }, + "vl-4-4-combined.concurrent.client.log": { + "bytes": 3232, + "sha256": "fb487abdb880c8100d647468a2cf1ebca567d4716787f6cd9c0c4a91e75bf57a" + }, + "vl-4-4-combined.concurrent.json": { + "bytes": 286399, + "sha256": "d1005b226f6a12988c7e3441d5570f188178485eb335ef9e2f9edbd3f704de9d" + }, + "vl-4-4-combined.server.log": { + "bytes": 13571, + "sha256": "330f27864d8bb54bb91a04f5b112da15748dd9af60d97e57aabccfb380b34978" + }, + "vl-4-5-public.after.health": { + "bytes": 298, + "sha256": "071cb00608c4dba9f3c96bf0fe786cbc380d73d9cabfbc2dac30a7f98ca06691" + }, + "vl-4-5-public.after.metrics": { + "bytes": 12303, + "sha256": "f0af2844d4dab7231a07111df4d8716b0e3d577c403fa0269606280bed6374c2" + }, + "vl-4-5-public.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-5-public.before.health": { + "bytes": 296, + "sha256": "8c789a277ca189a0923767fa83532a352de4a11334a37855d824cd2137333df4" + }, + "vl-4-5-public.before.metrics": { + "bytes": 12128, + "sha256": "d104ba544adcc2cfa419f70f318c739fa606ac56eb0f3dfd4040c51d55cd905c" + }, + "vl-4-5-public.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-4-5-public.cold-2048.client.log": { + "bytes": 348, + "sha256": "ff06617eb5a750af3d6ba6fabccede4ea9cf89359aecd654d82efbbabf06834b" + }, + "vl-4-5-public.cold-2048.json": { + "bytes": 20276, + "sha256": "0314819143b49e4b4382cb8e49df1885a14cf6b5a775b31a9a01e40199a5405d" + }, + "vl-4-5-public.cold-8192.client.log": { + "bytes": 348, + "sha256": "c2191169a10f24ae06903210d14a0da8f4c1f15166722f4f9a44a707172f1ad1" + }, + "vl-4-5-public.cold-8192.json": { + "bytes": 57149, + "sha256": "39ab1a8ca9e37da85f611bf75d71897c9f162b08760dd2cec2b221b9e5b95995" + }, + "vl-4-5-public.concurrent.client.log": { + "bytes": 3218, + "sha256": "8f713283ff444de7cde34b0e422f1c7e1f72ecbb7c4dda676d3d5af6839e5ac6" + }, + "vl-4-5-public.concurrent.json": { + "bytes": 286440, + "sha256": "b826b0e5373ccf1da367fa037707bf6c098506de020210d86abf428d28aed468" + }, + "vl-4-5-public.server.log": { + "bytes": 13576, + "sha256": "1b02113c52425afa7b408f453db6a168d8ab0ae2fc3cdca036d05f1fe434ce78" + } + }, + "raw_retention": "Raw client/server logs, health/metrics/settings captures, manifest, events and records remain in local OUTPUT. This receipt includes their hashes rather than copying large token timestamp arrays or unrelated model catalog entries." +} diff --git a/docs/receipts/lfm3/m5-cold-stage-profile.json b/docs/receipts/lfm3/m5-cold-stage-profile.json new file mode 100644 index 0000000000..c39bcf7526 --- /dev/null +++ b/docs/receipts/lfm3/m5-cold-stage-profile.json @@ -0,0 +1,559 @@ +{ + "scope": "M5 text4 exactHTTPcold2k8k diagnostic; existing profiler labels only, no kernel/arithmetic variant. Stage instrumentation may perturb schedules; plain wall clocks retained. No performance acceptance.", + "hardware": "M5; exclusive existing model lease; no all-width or serving performance acceptance", + "native_cache_capacity": 70000, + "native_prefill_chunk": 128, + "token_verification": [ + { + "tokens": 2048, + "actual_current_native_tokenize_matches_pinned_http_fixture": true, + "body_sha256": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "npy_sha256": "cc3c5a72ddfc5cd60f83fa046250606cc452a0ac483e49184626dc78e5a45d12" + }, + { + "tokens": 8192, + "actual_current_native_tokenize_matches_pinned_http_fixture": true, + "body_sha256": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "npy_sha256": "92bcc953dc435011aec99e723b25210e105f8b9cc4d370d938722ce3861d690a" + } + ], + "source_sha256": { + "zig/tests/lfm3_profile.zig": "f5620a0ca8565abb3d923c2da6793c1b51eb030fa87214d8a7f933195e17bb32", + "zig/src/core/gpu_profile.zig": "80eb55884ef6d7c9b894f06e3ebfa06a4a485bd3f48895d634ed89d6a638261b", + "zig/src/families/lfm3/forward.zig": "894c9541112518287bc9985db51f7e023b870b234da54acc626e9f2f57533bc9", + "zig/kernels/metal/lfm3/attention.metal": "6dc51acbb45e2e65028e72fed72d318629eb7b382e773c5837db9baa8de5d3b9", + "zig/kernels/metal/lfm3/tile_pair.metal": "e21ba279d7518093dd7a8e74bfb848da7f41d97136a2c2ad9d2a1c1d226be50f", + "zig/src/families/lfm3/prompt_projection.zig": "634d4f1ec738dad4a067410875760a66b790cc318e63b54ecf880f18bf67c235", + "zig/src/families/lfm3/tile_pair.zig": "d98e6de843242f0620b79f65568a0a0a859e00138ad807464f8e1d49f6fbed9e" + }, + "diagnostic_binary_sha256": "f664911c1ad88511208596165604bd142e8d689a6afb8889e1e44f88d04e4fb1", + "same_variant_profiled_and_plain_endpoint_identity": true, + "cross_variant_bit_identity_claimed": false, + "runs": [ + { + "tokens": 2048, + "variant": "current", + "cache_capacity": 70000, + "chunk": 128, + "counter_mode": [ + "stage", + "stage" + ], + "plain_wall_seconds_median": 0.883111, + "plain_gpu_seconds_median": 0.846759352600202, + "profiled_stage_seconds_sum": 0.851895154, + "stages": { + "conv_in": { + "gpu_seconds_median": 0.08091941450000001, + "calls_each": [ + 208, + 208 + ] + }, + "conv_out": { + "gpu_seconds_median": 0.028126116000000003, + "calls_each": [ + 208, + 208 + ] + }, + "attn_q": { + "gpu_seconds_median": 0.0237688785, + "calls_each": [ + 176, + 176 + ] + }, + "attn_kv": { + "gpu_seconds_median": 0.012734998, + "calls_each": [ + 176, + 176 + ] + }, + "attn_gate": { + "gpu_seconds_median": 0.001957206, + "calls_each": [ + 176, + 176 + ] + }, + "attn_out": { + "gpu_seconds_median": 0.0237685195, + "calls_each": [ + 176, + 176 + ] + }, + "ffn_gate_up": { + "gpu_seconds_median": 0.36674050300000005, + "calls_each": [ + 384, + 384 + ] + }, + "ffn_down": { + "gpu_seconds_median": 0.179886763, + "calls_each": [ + 384, + 384 + ] + }, + "vocabulary_head": { + "gpu_seconds_median": 0.0023307285000000004, + "calls_each": [ + 16, + 16 + ] + }, + "norm": { + "gpu_seconds_median": 0.0042161035, + "calls_each": [ + 784, + 784 + ] + }, + "embedding": { + "gpu_seconds_median": 0.00010474999999999998, + "calls_each": [ + 16, + 16 + ] + }, + "activation": { + "gpu_seconds_median": 0.005800944500000001, + "calls_each": [ + 768, + 768 + ] + }, + "attention_prepare": { + "gpu_seconds_median": 0.0012115645, + "calls_each": [ + 176, + 176 + ] + }, + "attention": { + "gpu_seconds_median": 0.12032866449999999, + "calls_each": [ + 704, + 704 + ] + } + }, + "all_profiled_endpoints_equal_plain": true, + "repeat_final_logit_sha256": "8700e9a12456dce4f744caa329ad2013f44df7f2704e58e245e52aa0e9e9ddc5", + "attention_fraction_of_profiled_stage_time": 0.1412482086968181, + "plain_wall_seconds_each": [ + 0.882642, + 0.88358 + ] + }, + { + "tokens": 2048, + "variant": "paired_tiles", + "cache_capacity": 70000, + "chunk": 128, + "counter_mode": [ + "stage", + "stage" + ], + "plain_wall_seconds_median": 0.46441299999999996, + "plain_gpu_seconds_median": 0.42822166811674833, + "profiled_stage_seconds_sum": 0.438255828, + "stages": { + "conv_in": { + "gpu_seconds_median": 0.0322938525, + "calls_each": [ + 208, + 208 + ] + }, + "conv_out": { + "gpu_seconds_median": 0.013225664500000001, + "calls_each": [ + 208, + 208 + ] + }, + "attn_q": { + "gpu_seconds_median": 0.011144940500000002, + "calls_each": [ + 176, + 176 + ] + }, + "attn_kv": { + "gpu_seconds_median": 0.007488039, + "calls_each": [ + 176, + 176 + ] + }, + "attn_gate": { + "gpu_seconds_median": 0.0020158750000000003, + "calls_each": [ + 176, + 176 + ] + }, + "attn_out": { + "gpu_seconds_median": 0.011178397000000003, + "calls_each": [ + 176, + 176 + ] + }, + "ffn_gate_up": { + "gpu_seconds_median": 0.1439456705, + "calls_each": [ + 384, + 384 + ] + }, + "ffn_down": { + "gpu_seconds_median": 0.08293642150000001, + "calls_each": [ + 384, + 384 + ] + }, + "vocabulary_head": { + "gpu_seconds_median": 0.0023206685000000003, + "calls_each": [ + 16, + 16 + ] + }, + "norm": { + "gpu_seconds_median": 0.004304714, + "calls_each": [ + 784, + 784 + ] + }, + "embedding": { + "gpu_seconds_median": 0.000251833, + "calls_each": [ + 16, + 16 + ] + }, + "activation": { + "gpu_seconds_median": 0.005752878499999999, + "calls_each": [ + 768, + 768 + ] + }, + "attention_prepare": { + "gpu_seconds_median": 0.0012094789999999998, + "calls_each": [ + 176, + 176 + ] + }, + "attention": { + "gpu_seconds_median": 0.12018739449999999, + "calls_each": [ + 704, + 704 + ] + } + }, + "all_profiled_endpoints_equal_plain": true, + "repeat_final_logit_sha256": "7ad8423d5e79a75bc4b72f9a0b32b4f2023272d3a1ec04b597fb37cd5438e964", + "attention_fraction_of_profiled_stage_time": 0.2742402652087492, + "plain_wall_seconds_each": [ + 0.4644, + 0.464426 + ] + }, + { + "tokens": 8192, + "variant": "current", + "cache_capacity": 70000, + "chunk": 128, + "counter_mode": [ + "stage", + "stage" + ], + "plain_wall_seconds_median": 4.638943, + "plain_gpu_seconds_median": 4.5858032503165305, + "profiled_stage_seconds_sum": 4.605206704, + "stages": { + "conv_in": { + "gpu_seconds_median": 0.32363039149999995, + "calls_each": [ + 832, + 832 + ] + }, + "conv_out": { + "gpu_seconds_median": 0.11234039850000002, + "calls_each": [ + 832, + 832 + ] + }, + "attn_q": { + "gpu_seconds_median": 0.09514477200000002, + "calls_each": [ + 704, + 704 + ] + }, + "attn_kv": { + "gpu_seconds_median": 0.05088421800000002, + "calls_each": [ + 704, + 704 + ] + }, + "attn_gate": { + "gpu_seconds_median": 0.0079379555, + "calls_each": [ + 704, + 704 + ] + }, + "attn_out": { + "gpu_seconds_median": 0.09525006349999998, + "calls_each": [ + 704, + 704 + ] + }, + "ffn_gate_up": { + "gpu_seconds_median": 1.4671723220000001, + "calls_each": [ + 1536, + 1536 + ] + }, + "ffn_down": { + "gpu_seconds_median": 0.719726452, + "calls_each": [ + 1536, + 1536 + ] + }, + "vocabulary_head": { + "gpu_seconds_median": 0.009268914500000003, + "calls_each": [ + 64, + 64 + ] + }, + "norm": { + "gpu_seconds_median": 0.0169286425, + "calls_each": [ + 3136, + 3136 + ] + }, + "embedding": { + "gpu_seconds_median": 0.00036797999999999996, + "calls_each": [ + 64, + 64 + ] + }, + "activation": { + "gpu_seconds_median": 0.023302213, + "calls_each": [ + 3072, + 3072 + ] + }, + "attention_prepare": { + "gpu_seconds_median": 0.0048518125, + "calls_each": [ + 704, + 704 + ] + }, + "attention": { + "gpu_seconds_median": 1.6784005685000003, + "calls_each": [ + 2816, + 2816 + ] + } + }, + "all_profiled_endpoints_equal_plain": true, + "repeat_final_logit_sha256": "19a2f9b7b0794ef2f2be0edb7d16564e24d33f02e0d01972780ae28d09010bf5", + "attention_fraction_of_profiled_stage_time": 0.36445716259428085, + "plain_wall_seconds_each": [ + 4.63588, + 4.642006 + ] + }, + { + "tokens": 8192, + "variant": "paired_tiles", + "cache_capacity": 70000, + "chunk": 128, + "counter_mode": [ + "stage", + "stage" + ], + "plain_wall_seconds_median": 3.107767, + "plain_gpu_seconds_median": 3.053785268450156, + "profiled_stage_seconds_sum": 3.064434951, + "stages": { + "conv_in": { + "gpu_seconds_median": 0.1291200075, + "calls_each": [ + 832, + 832 + ] + }, + "conv_out": { + "gpu_seconds_median": 0.05291137350000001, + "calls_each": [ + 832, + 832 + ] + }, + "attn_q": { + "gpu_seconds_median": 0.04456602450000001, + "calls_each": [ + 704, + 704 + ] + }, + "attn_kv": { + "gpu_seconds_median": 0.029950899500000003, + "calls_each": [ + 704, + 704 + ] + }, + "attn_gate": { + "gpu_seconds_median": 0.0081318085, + "calls_each": [ + 704, + 704 + ] + }, + "attn_out": { + "gpu_seconds_median": 0.04479231750000002, + "calls_each": [ + 704, + 704 + ] + }, + "ffn_gate_up": { + "gpu_seconds_median": 0.5757702760000001, + "calls_each": [ + 1536, + 1536 + ] + }, + "ffn_down": { + "gpu_seconds_median": 0.331787373, + "calls_each": [ + 1536, + 1536 + ] + }, + "vocabulary_head": { + "gpu_seconds_median": 0.009284855000000002, + "calls_each": [ + 64, + 64 + ] + }, + "norm": { + "gpu_seconds_median": 0.017096581500000003, + "calls_each": [ + 3136, + 3136 + ] + }, + "embedding": { + "gpu_seconds_median": 0.0008800625000000004, + "calls_each": [ + 64, + 64 + ] + }, + "activation": { + "gpu_seconds_median": 0.0231809975, + "calls_each": [ + 3072, + 3072 + ] + }, + "attention_prepare": { + "gpu_seconds_median": 0.004806967500000002, + "calls_each": [ + 704, + 704 + ] + }, + "attention": { + "gpu_seconds_median": 1.7921554069999999, + "calls_each": [ + 2816, + 2816 + ] + } + }, + "all_profiled_endpoints_equal_plain": true, + "repeat_final_logit_sha256": "305e345e11883986154d8b8dde198985589ac1d064517c14bdd46224c46dd074", + "attention_fraction_of_profiled_stage_time": 0.5848240982942633, + "plain_wall_seconds_each": [ + 3.114677, + 3.100857 + ] + } + ], + "raw_logical_limits": [ + "These are prefill-only direct native passes, not16-output HTTP elapsed time.", + "Paired FFN gate/up and K/V remain grouped because each is a single existing dispatch.", + "Stage counter instrumentation can perturb scheduling; matched unprofiled wall clocks retained.", + "Projection class labels are host metadata; all kernels/arithmetic/dispatches unchanged." + ], + "performance_accepted": false, + "dominant_eight_k": { + "paired_tiles_attention_seconds": 1.792155407, + "paired_tiles_attention_fraction": 0.5848240982942633, + "paired_tiles_ffn_gate_up_plus_down_seconds": 0.9075576489999999, + "interpretation": "Shared tiles reduce dense time; scalar attention is the largest remaining8k stage and FFN remains substantial. No unmeasured standard stage breakdown inferred." + }, + "counter_clock_scope": { + "counter_mode": "stage; calibrated GPU counter samples for all8runs", + "profiled_command_buffer_gpu_timestamps": "All8invalid/unavailable clocks discarded (null inraw), never used forstage totals.", + "plain_clocks": "Uninstrumented wall and valid plaincommandGPUtimes retained separately; profiledstage sum is diagnostic, notHTTPelapsedtime." + }, + "raw_evidence_sha256": { + "cold-stage-profile/0-current-2048.json": "d57819311ac26762671e0741fb2652cf606000b0c7036cddc42b6b304504bb8e", + "cold-stage-profile/0-current-2048.log": "3abf7120a89d3ff17e97e5d25061c4199236419aba394e4cada6f084877cc954", + "cold-stage-profile/1-paired-2048.json": "2f17f9e7044183412c7a52d3067c1c761398c9df5bf2499c0aae9622b5b64142", + "cold-stage-profile/1-paired-2048.log": "3abf7120a89d3ff17e97e5d25061c4199236419aba394e4cada6f084877cc954", + "cold-stage-profile/2-paired-8192.json": "6f126f49b939dcbf7207afa1f9bc03696f2661a3a4ce39072923bdbb20c5589b", + "cold-stage-profile/2-paired-8192.log": "5cf360bde5c1a9e0ff68b1cbc385982da901a621793fe54b8ef9fac5763c83ef", + "cold-stage-profile/3-current-8192.json": "98492b420fe61647cef131e6b661e663c51bcfcd3ba8b904ac45c3b913c7fa02", + "cold-stage-profile/3-current-8192.log": "5cf360bde5c1a9e0ff68b1cbc385982da901a621793fe54b8ef9fac5763c83ef", + "cold-stage-profile/4-current-8192.json": "280bac4b3abd0f17d2a233fd7946292e500f970af72fc6b76b09650454b59d13", + "cold-stage-profile/4-current-8192.log": "5cf360bde5c1a9e0ff68b1cbc385982da901a621793fe54b8ef9fac5763c83ef", + "cold-stage-profile/5-paired-8192.json": "a6d2afadee39dd978e5a4b61d3dbceeda905bf09f4bc809c84a4ea9cc378c00b", + "cold-stage-profile/5-paired-8192.log": "5cf360bde5c1a9e0ff68b1cbc385982da901a621793fe54b8ef9fac5763c83ef", + "cold-stage-profile/6-paired-2048.json": "df474ed6d68b6065d36ac6486c5237abb93d86dfdbe0ad356f47c09ab0c126fb", + "cold-stage-profile/6-paired-2048.log": "3abf7120a89d3ff17e97e5d25061c4199236419aba394e4cada6f084877cc954", + "cold-stage-profile/7-current-2048.json": "8028346486e0fd8ef4c736f9d688573edac1b9c2c6166e1a4ab241570b6794aa", + "cold-stage-profile/7-current-2048.log": "3abf7120a89d3ff17e97e5d25061c4199236419aba394e4cada6f084877cc954", + "cold-stage-profile/admission.json": "be0c3a3290919ede0ec7782068bd9eb8582d6d3bc309bac481f40f0814eec171", + "cold-stage-profile/events.json": "5d28399bcaf6fecc70c6559fcae7f4d475f33f0fdecb45503d3dfc1c825b3d76", + "cold-stage-profile/records.json": "59fe919a83146c0ac1d9564f4cbf5805666b08719d9994de1bfe3f23c49f1f17", + "cold-stage-profile/summary.json": "8e939eda43fc2fe0624cfdbf002811e394706d954f6d4fbd3aa35f6e1487bdf7", + "cold-token-capture/events.json": "e40beeedfd304799b8e4ee137d00903873008b03befa7a729f0e39c3f9a68dd9", + "cold-token-capture/receipt.json": "f9c67f355260b8a706147ce1d27d4765c3e18f265d12343d4b877b8ee10cfc53", + "cold-token-capture/text-2048.npy": "cc3c5a72ddfc5cd60f83fa046250606cc452a0ac483e49184626dc78e5a45d12", + "cold-token-capture/text-8192.npy": "92bcc953dc435011aec99e723b25210e105f8b9cc4d370d938722ce3861d690a", + "cold-token-capture/text-capture.server.log": "3792e9007dc40ec42809781d62f6a3f7d8a1c15f96f79129e995a56a83403872" + } +} diff --git a/docs/receipts/lfm3/m5-current-pair-standard.json b/docs/receipts/lfm3/m5-current-pair-standard.json new file mode 100644 index 0000000000..1b137f1eb1 --- /dev/null +++ b/docs/receipts/lfm3/m5-current-pair-standard.json @@ -0,0 +1,6697 @@ +{ + "schema_version": 1, + "scope": "Fresh exclusive guarded Apple M5 Ultra screen: current public versus paired GS64 tile/twin candidate and pinned standard for TEXT4; current public versus pinned standard for VL4. No GS32 VL candidate in this screen.", + "status": { + "actual_exit": 0, + "server_legs": 10, + "measured_decode_requests": 300, + "measured_decode_repetitions": 120, + "cold_requests": 20, + "strict_decode_tokens": 256, + "strict_cold_output_tokens": 16, + "cached_tokens": 0, + "all_lengths_validated": true, + "owned_server_cleanups": 10, + "performance_accepted": false, + "performance_conclusion": "Neither current public nor paired candidate meets the measured standard throughput/prefill comparison. Paired tiles improve cold prefill versus current public but regress M5 decode. M4 gains are not a M5 decode win." + }, + "correctness_prerequisites": { + "actual_exit": 0, + "current_public_registered_kernels": 70, + "known_findings": 0, + "failures": 0, + "controls_red": 2, + "limit_problems": 0, + "shared_tile_twin_kernels_passed": 6, + "paired_raw_cases": 108 + }, + "host": { + "gpu": "Apple M5 Ultra", + "gpu_cores": 64, + "physical_memory_bytes": 274877906944, + "os": "ProductName:\t\tmacOS\nProductVersion:\t\t27.0\nBuildVersion:\t\t26A428\n" + }, + "provenance": { + "native_source_head_at_preparation": "6a111de5c1b4ef5bcb60d2924446636733beb416", + "native_provenance": "Current artifact3d4 was built before later receipt-only commits, from applied source hashes inattention-range.json; runtime remains identical at preparation HEAD6a111de5. This is not a rebuildat6a. Runtimefixdd7263c26; receiptcommitd3c52154. Pair overlaybasedpublicd3c, exactfilespinned tilepair-manifest.json. No sumcache composition.", + "native_artifacts": { + "current": { + "path": "$RUN/bin/tensorfold-native-current", + "sha256": "3d4a3daeda697420bfbba6d81972896c2bf9e31efaf6f12321ee8487a299354b" + }, + "pair": { + "path": "$RUN/bin/tensorfold-native-pair", + "sha256": "464f472eb8e8bd85316de7122a86f4bcd169c24e86f87d3e469cb181f87b538f" + } + }, + "proof_artifacts": { + "tf-lfm3-dual-project": "dc34ace5c6e0055eb6cb127aabbfdf61fd45245697502dfa0e987ea668d637a2", + "tensorfold-native-current": "3d4a3daeda697420bfbba6d81972896c2bf9e31efaf6f12321ee8487a299354b", + "tf-kernel-contract": "2786719bfe0314754349463721b8e1faa7cd06f6f2efa142db19acacd83d54e6", + "tensorfold-native-pair": "464f472eb8e8bd85316de7122a86f4bcd169c24e86f87d3e469cb181f87b538f" + }, + "standard": { + "python": "$QUALIFICATION/standard-venv/bin/python", + "module": "mlx_vlm.server", + "source_commit": "230fc308efb577d87446306bfb8a3e9f1a129558", + "mlx": "0.32.4", + "mlx_metal": "0.32.4", + "mlx_vlm": "0.7.11", + "dependency_provenance": "Existing remote environment, installedmetadata captured CPUonly; all distributions frozen in requirements.lock. CoreMLX/VLMpins matchsource, dependency differences fromlocalM4retained.", + "memory_policy": "Unmodified MLX defaults; no memory/wired limit overrides.", + "request_context_limit": 70000, + "context_policy": "Pinned --max-kv-size70000 sets request budget; LFM3 family cache construction remains lazy and is not claimed allocation-identical to native.", + "checkpoint_context_caps": { + "text": 262144, + "vl": 128000 + } + }, + "versions": { + "python": "3.13.15 (main, Sep 1 2026, 14:05:45) [Clang 22.1.3 ]", + "packages": { + "shellingham": "1.5.4", + "uvicorn": "0.54.0", + "xxhash": "4.0.2", + "filelock": "4.2.4", + "mlx": "0.32.4", + "mlx-lm": "0.32.2", + "starlette": "1.10.0", + "huggingface_hub": "2.3.0", + "click": "8.5.0", + "packaging": "26.3", + "protobuf": "7.36.2", + "tokenizers": "0.23.3", + "Pygments": "2.21.0", + "pyarrow": "26.0.0", + "sounddevice": "0.5.6", + "requests": "2.34.2", + "rich": "15.0.0", + "certifi": "2026.10.23", + "yarl": "1.25.1", + "websockets": "17.3", + "mlx-vlm": "0.7.11", + "opentelemetry-api": "1.45.1", + "pandas": "3.1.0", + "datasets": "5.1.0", + "sentencepiece": "0.2.2", + "propcache": "0.5.4", + "llguidance": "1.9.1", + "numpy": "2.5.4", + "aiohappyeyeballs": "2.7.1", + "cryptography": "50.0.2", + "pycparser": "3.12", + "tzdata": "2026.5", + "pillow": "13.0.0", + "fsspec": "2026.7.0", + "hf-xet": "1.7.0", + "safetensors": "0.8.0", + "miniaudio": "1.71", + "tqdm": "4.70.1", + "urllib3": "2.8.0", + "h11": "0.16.0", + "pydantic_core": "2.50.1", + "mdurl": "0.1.2", + "python-multipart": "0.0.32", + "mlx-metal": "0.32.4", + "PyYAML": "6.0.3", + "httpcore2": "2.16.0", + "opencv-python": "5.0.0.93", + "truststore": "0.10.5", + "frozenlist": "1.8.0", + "cffi": "2.1.1", + "httpx2": "2.16.0", + "aiosignal": "1.4.0", + "typer": "0.27.3", + "mlx-audio": "0.6.2", + "Jinja2": "3.1.6", + "annotated-types": "0.8.0", + "regex": "2026.9.29", + "markdown-it-py": "4.2.0", + "annotated-doc": "0.0.5", + "typing-inspection": "0.4.4", + "fastapi": "0.162.0", + "aiohttp": "3.14.5", + "attrs": "26.1.0", + "multidict": "7.2.1", + "typing_extensions": "4.16.0", + "MarkupSafe": "3.0.4", + "pydantic": "2.14.1", + "python-dateutil": "3.0.0", + "multiprocess": "0.70.19", + "charset-normalizer": "3.5.2", + "anyio": "4.15.1", + "six": "1.17.0", + "scipy": "1.18.1", + "dill": "0.4.1", + "transformers": "5.21.0", + "idna": "3.20" + } + }, + "requirements_sha256": "275b369ea72d5757b1e3f91e68ed5b746d83b5f7667adc8f5d9d28bb8933ddd6", + "models": { + "LFM3-1.7B-MLX-4bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-4bit", + "revision": "d0ca73dcfa70ce18733754a492cdc1fb0123262d", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 2423, + "sha256": "7df2ea819f9a31b770192a0a9dc1b7953a7dd8c41b816cc3905bbec99ef332bd", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 328, + "sha256": "0095ba7158e938b1e2d3082d412f00d8be85e42cbbd7608b63866f7a081cf270", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 973083386, + "sha256": "1be5bf672013cd91aa62148f77c5eea8d2784a95845d5fe826e08c7b84823815", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 50700, + "sha256": "2c8b88e25f6a1055164dd051436b170872200047bce4b3ec5f2a58f6ef929a75", + "verified_against_pinned_revision": true + }, + "special_tokens_map.json": { + "bytes": 92, + "sha256": "7946f864cbfb62a1b041bda937244210a1a87f4a6cab3a48b791131b2cd57f88", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908634, + "sha256": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1859, + "sha256": "b77214b9f12be74b9e5e1dab35d7dace077a31be969314bf38866fd6c8163853", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-VL-2B-MLX-4bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "revision": "c823fade98e35a3d6942ad4d38b1ea15f675cd12", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 4419, + "sha256": "0efb7708e0771e3e104ac628954cdc71070dd436e0b7974615bd1b9eb836f9d9", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 323, + "sha256": "30b1115bcdc4e5671567a5b1255a2f3d2f5a912a79292051c6f1e9db81b5e8c4", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1590959088, + "sha256": "58268dfdf441ae4020d2a52fc97a814645e8a9624946e2d490403d4dcd71a255", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 61256, + "sha256": "38b756dbde167ab7c60756af7d5652414c31afdf4e77700b8d3c99395515d639", + "verified_against_pinned_revision": true + }, + "processor_config.json": { + "bytes": 1254, + "sha256": "f183f5a5e5b3a32e666fce05033e6cb7ca496d26e709a2cee611a13eb7d6b610", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908258, + "sha256": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1759, + "sha256": "d63d1ce2f332047e398764b4170352b96629a5194643105bfd6d70c90b750396", + "verified_against_pinned_revision": true + } + } + } + }, + "clients_sha256": { + "$PACKET/tools/bench_auth.py": "1896120b70d0df05646870a252697e20ce84c63e09dcba88b2a4e5a55ccb3c5b", + "$PACKET/tools/bench_cache.py": "24364dd13c033450f45710fc8685e2c64d6a72d6dd6e223d0d7b19aabbdc1d98", + "$PACKET/tools/bench_concurrent.py": "0e72e26769f082459329fc068c25f6f72a9155e422962ea54cb5d1226227752d", + "$PACKET/tools/bench_evidence.py": "54c28474f0804dbd120e3fb5859be0d1667ef21ca502880d0bed098ec9bb42ff", + "$PACKET/tools/bench_metrics.py": "26f4c3ecbb03df4f12e24d18840fea989d9ef8c63117ca11eb944ef670458a27", + "$PACKET/tools/bench_openai.py": "d5af4fbb756cdfb573546c0e0017652a4bb8f509d1251af683be7a3cbdd426f6", + "$PACKET/tools/bench_timing.py": "158f636ef0b3bed460ce19ef4872b6d8607ff7a8590562d06bae8bd55e6f44ed", + "$PACKET/tools/lfm3/bench_decode.py": "c011106f4f182db19371e3f91f70bcc374160786dd105f2d713a7fea07ab8c4c", + "$PACKET/cold_client.py": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2" + }, + "fixtures_sha256": { + "text-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "text-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "text-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "text-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd", + "vl-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "vl-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "vl-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "vl-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd" + }, + "provenance_files": { + "tilepair-manifest.json": "a6b5e4cbb9c7f96f6e95bf7d03366b022d123af653bb2e01edd9acd487a02528", + "current-source-receipt.json": "b04753edfcec89c76161e2c3e7e9ea873700179dee7d33c937e2cc53e2f0f654", + "standard-installed-metadata.json": "fb244f662eddced712fac29a25e390dad3d28f0cec79a5284f1c63cfb875b4e8" + }, + "executed_packet_files_sha256": { + "run_existing_clients.py": "8209512004d5c8c19e77a4a1ce1032b54e492e8625faf448cfa34a07e268a2c0", + "guard.py": "b37fb2b0aa20bc29e0d7f7bb4930ce267752c26172c9bb368b2ab18a7392954d", + "config.json": "de8ef0209c0e828118a2aee2442ae1a80a59b2fc5902dc0cec0e3cbbae82aaa7", + "requirements.lock": "275b369ea72d5757b1e3f91e68ed5b746d83b5f7667adc8f5d9d28bb8933ddd6", + "packet-sha256.json": "f3b1b02a9652d83dab8a721d133bbde57d1964c5d7c34f6572c15e9ba6f70e74" + }, + "packet_hash_check": "Read remote packet files after completedrun; hashes exactlymatchfrozenpreparedpacket.", + "public_runtime_source_commit": "dd7263c26", + "public_runtime_same_at_receipt_commit": "6a111de5c1b4ef5bcb60d2924446636733beb416" + }, + "protocol": { + "phase": { + "families": [ + "text", + "vl" + ], + "bits": [ + 4 + ], + "cold_tokens": [ + 2048, + 8192 + ], + "decode_tokens": 256, + "cold_output_tokens": 16, + "reps": 3, + "levels": [ + 1, + 4 + ], + "order": "family_orders", + "family_orders": { + "text": [ + "current", + "pair", + "standard", + "standard", + "pair", + "current" + ], + "vl": [ + "current", + "standard", + "standard", + "current" + ] + } + }, + "settings": { + "native_context": 70000, + "parallel": 4, + "prompt_cache_gib": 0, + "temperature": 0, + "thinking": false, + "prefix_cache": "disabled on both", + "port": 18085 + }, + "lock_path": "/tmp/tensorfold-model.lock", + "guard_source": { + "path": "guard.py", + "sha256": "b37fb2b0aa20bc29e0d7f7bb4930ce267752c26172c9bb368b2ab18a7392954d", + "origin": "ExistingM5guard exactaliases supplied byqualificationowner; no weakened admission." + }, + "http_lock_events": [ + { + "event": "lock_wait", + "time_ns": 1793200491705862000, + "path": "/tmp/tensorfold-model.lock" + }, + { + "event": "lock_acquired", + "time_ns": 1793200491706056000, + "path": "/tmp/tensorfold-model.lock", + "pid": 14813, + "device": 16777233, + "inode": 2622134 + }, + { + "event": "complete", + "time_ns": 1793200708650218000, + "legs": 10, + "all_lengths_validated": true, + "performance_gate_claim": false + }, + { + "event": "lock_release", + "time_ns": 1793200708650385000, + "pid": 14813 + } + ], + "single_model_admission": "Existing exclusive flock plus exact process admission before load and before/after clients and cleanup. Only owned descendants allowed during measurement; no foreign model killed.", + "native_admission": { + "hardware": "M5", + "binary_sha256": { + "tf-lfm3-dual-project": "dc34ace5c6e0055eb6cb127aabbfdf61fd45245697502dfa0e987ea668d637a2", + "tensorfold-native-current": "3d4a3daeda697420bfbba6d81972896c2bf9e31efaf6f12321ee8487a299354b", + "tf-kernel-contract": "2786719bfe0314754349463721b8e1faa7cd06f6f2efa142db19acacd83d54e6", + "tensorfold-native-pair": "464f472eb8e8bd85316de7122a86f4bcd169c24e86f87d3e469cb181f87b538f" + }, + "memory_admission": { + "4": { + "checkpoint_bytes": 973083386, + "conservative_scratch_plus_existing_2GiB_margin_bytes": 4294967296, + "estimated_total_bytes": 5268050682, + "existing_70percent_ram_policy_bytes": 192414534830, + "no_limits_changed": true + }, + "6": { + "checkpoint_bytes": 1405187179, + "conservative_scratch_plus_existing_2GiB_margin_bytes": 4294967296, + "estimated_total_bytes": 5700154475, + "existing_70percent_ram_policy_bytes": 192414534830, + "no_limits_changed": true + }, + "8": { + "checkpoint_bytes": 1837290805, + "conservative_scratch_plus_existing_2GiB_margin_bytes": 4294967296, + "estimated_total_bytes": 6132258101, + "existing_70percent_ram_policy_bytes": 192414534830, + "no_limits_changed": true + } + }, + "model_budget_scope": "Fixed raw projection inputs <=128rows plus conservative2GiB scratch and existing2GiB margin; native serving retains its actual Metal admission", + "guard": "existing exclusive /tmp/tensorfold-model.lock and exact process admission; no prior read-only probe claimed as lease" + }, + "memory_policy": "Native admission unchanged; MLX memory and wired limits unmodified. /usr/bin/time -l captures physical footprint and RSS; available server metrics preserved. No allocator/pinned peak inferred from RSS.", + "server_commands": [ + { + "tag": "text-4-0-current", + "command": [ + "$RUN/bin/tensorfold-native-current", + "serve", + "$QUALIFICATION/LFM3-1.7B-MLX-4bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "tag": "text-4-1-pair", + "command": [ + "$RUN/bin/tensorfold-native-pair", + "serve", + "$QUALIFICATION/LFM3-1.7B-MLX-4bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "tag": "text-4-2-standard", + "command": [ + "$QUALIFICATION/standard-venv/bin/python", + "-m", + "mlx_vlm.server", + "--model", + "$QUALIFICATION/LFM3-1.7B-MLX-4bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--max-tokens", + "256", + "--max-num-seqs", + "4", + "--max-kv-size", + "70000", + "--vision-cache-size", + "0", + "--log-progress-interval", + "0" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": "70000" + }, + { + "tag": "vl-4-1-standard", + "command": [ + "$QUALIFICATION/standard-venv/bin/python", + "-m", + "mlx_vlm.server", + "--model", + "$QUALIFICATION/LFM3-VL-2B-MLX-4bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--max-tokens", + "256", + "--max-num-seqs", + "4", + "--max-kv-size", + "70000", + "--vision-cache-size", + "0", + "--log-progress-interval", + "0" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": "70000" + } + ], + "actual_dispatch": [ + { + "leg": "text-4-1-pair", + "actual_server_log_line": "info: lfm3 tile qualification: gs64_tiles=true twin=true gs32_affine=false", + "server_log_sha256": "ccf6078e0f3167f216f2cf23ab55175debdcbb9532fa5749d332b1d2e0f4776b" + }, + { + "leg": "text-4-4-pair", + "actual_server_log_line": "info: lfm3 tile qualification: gs64_tiles=true twin=true gs32_affine=false", + "server_log_sha256": "2d899eb63007382b348bb13a70a245792633042bfe2ebc984bece39a752bc072" + } + ], + "replay": [ + "$QUALIFICATION/standard-venv/bin/python \"$PACKET/run_existing_clients.py\"", + "$QUALIFICATION/standard-venv/bin/python \"$PACKET/run_existing_clients.py\" --execute --output \"$NEW_OUTPUT\"" + ], + "replay_scope": "Existing locally retained frozen packet, artifact/model pins and clients are required; output directory must not exist. No new timing framework or throughput formula. Packet source and every retained raw result are hashed below." + }, + "results": { + "reported_summary": { + "text": { + "current": { + "median_of_two_reported_leg_cell_medians_tps": { + "long-code-chat/1": 315.75, + "long-code-chat/4": 861.85, + "long-prose-chat/1": 329.1, + "long-prose-chat/4": 878.75 + }, + "cold_median_ttft_s": { + "2048": 0.8591580830980092, + "8192": 4.615810395916924 + } + }, + "pair": { + "median_of_two_reported_leg_cell_medians_tps": { + "long-code-chat/1": 290.55, + "long-code-chat/4": 803.55, + "long-prose-chat/1": 303.45000000000005, + "long-prose-chat/4": 821.85 + }, + "cold_median_ttft_s": { + "2048": 0.4481715620495379, + "8192": 3.094886645907536 + } + }, + "standard": { + "median_of_two_reported_leg_cell_medians_tps": { + "long-code-chat/1": 344.2, + "long-code-chat/4": 995.0, + "long-prose-chat/1": 345.45, + "long-prose-chat/4": 1008.5 + }, + "cold_median_ttft_s": { + "2048": 0.14974241657182574, + "8192": 0.36902416590601206 + } + } + }, + "vl": { + "current": { + "median_of_two_reported_leg_cell_medians_tps": { + "long-code-chat/1": 298.9, + "long-code-chat/4": 835.65, + "long-prose-chat/1": 296.2, + "long-prose-chat/4": 831.3 + }, + "cold_median_ttft_s": { + "2048": 0.8618560209870338, + "8192": 4.652502874843776 + } + }, + "standard": { + "median_of_two_reported_leg_cell_medians_tps": { + "long-code-chat/1": 325.79999999999995, + "long-code-chat/4": 947.75, + "long-prose-chat/1": 325.25, + "long-prose-chat/4": 948.2 + }, + "cold_median_ttft_s": { + "2048": 0.15912006190046668, + "8192": 0.3741248541045934 + } + } + } + }, + "summary_method": "Each reported aggregate is the median of two existing per-leg cell medians (rounded by the unchanged client); all six original per-repetition TPS values remain below. Cold median uses two fresh-server leg samples per engine.", + "legs": [ + { + "family": "text", + "bits": 4, + "leg": 0, + "engine": "current", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.8591347499750555, + "output_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.614120041951537, + "output_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1215660032, + "peak_physical_footprint_bytes": 1630996160 + }, + "tag": "text-4-0-current", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 316.5, + "per_stream_tps": 316.5, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 309.67242448631606, + "makespan_s": 0.8266800004057586, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 316.84367346565756, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8257415001280606, + "end_to_end_tps": 310.0243840963523, + "timing_valid": true, + "per_stream_tps": [ + 316.8 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 315.4160036609081, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8293848331086338, + "end_to_end_tps": 308.6625047632971, + "timing_valid": true, + "per_stream_tps": [ + 315.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 316.48181437739015, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8266800004057586, + "end_to_end_tps": 309.67242448631606, + "timing_valid": true, + "per_stream_tps": [ + 316.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd" + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 862.2, + "per_stream_tps": 221.4, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 849.9678149898496, + "makespan_s": 1.204751499928534, + "steady_tps": 908.4, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 862.8268500146881, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2038995418697596, + "end_to_end_tps": 850.5693078092213, + "timing_valid": true, + "steady_tps": 908.9514539313226, + "per_stream_tps": [ + 219.5, + 223.4, + 227.5, + 215.7 + ], + "ttft_s": [ + 0.04, + 0.06, + 0.08, + 0.02 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 862.0828999413003, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2050185408443213, + "end_to_end_tps": 849.779455909876, + "timing_valid": true, + "steady_tps": 908.3430272058115, + "per_stream_tps": [ + 219.3, + 223.3, + 227.3, + 215.5 + ], + "ttft_s": [ + 0.04, + 0.06, + 0.08, + 0.02 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 862.2058417185594, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.204751499928534, + "end_to_end_tps": 849.9678149898496, + "timing_valid": true, + "steady_tps": 908.3991453698418, + "per_stream_tps": [ + 215.6, + 219.4, + 227.3, + 223.3 + ], + "ttft_s": [ + 0.02, + 0.04, + 0.08, + 0.06 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 329.8, + "per_stream_tps": 329.8, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 322.0194140045401, + "makespan_s": 0.7949831248261034, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 329.77222441181453, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.794658042024821, + "end_to_end_tps": 322.15114736359, + "timing_valid": true, + "per_stream_tps": [ + 329.8 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "555fe37e25c90f95" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 329.44161592865686, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7954670838080347, + "end_to_end_tps": 321.8234986851812, + "timing_valid": true, + "per_stream_tps": [ + 329.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "555fe37e25c90f95" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 329.75646324909707, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7949831248261034, + "end_to_end_tps": 322.0194140045401, + "timing_valid": true, + "per_stream_tps": [ + 329.8 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "555fe37e25c90f95" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 877.4, + "per_stream_tps": 225.3, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 864.6382683735401, + "makespan_s": 1.1843102918937802, + "steady_tps": 926.7, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 876.052712745667, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.186641083098948, + "end_to_end_tps": 862.9399525977931, + "timing_valid": true, + "steady_tps": 924.9855873783205, + "per_stream_tps": [ + 227.2, + 219.0, + 223.0, + 231.5 + ], + "ttft_s": [ + 0.06, + 0.02, + 0.04, + 0.08 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "555fe37e25c90f95", + "555fe37e25c90f95", + "555fe37e25c90f95", + "555fe37e25c90f95" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 877.4442938083836, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1841929159127176, + "end_to_end_tps": 864.7239704273617, + "timing_valid": true, + "steady_tps": 926.8623277669288, + "per_stream_tps": [ + 219.5, + 231.9, + 223.5, + 227.6 + ], + "ttft_s": [ + 0.02, + 0.08, + 0.04, + 0.06 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "555fe37e25c90f95", + "555fe37e25c90f95", + "555fe37e25c90f95", + "555fe37e25c90f95" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 877.5062881649212, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1843102918937802, + "end_to_end_tps": 864.6382683735401, + "timing_valid": true, + "steady_tps": 926.7290215431901, + "per_stream_tps": [ + 231.9, + 219.4, + 227.6, + 223.4 + ], + "ttft_s": [ + 0.08, + 0.02, + 0.06, + 0.04 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "555fe37e25c90f95", + "555fe37e25c90f95", + "555fe37e25c90f95", + "555fe37e25c90f95" + ] + } + ] + } + ] + }, + { + "family": "text", + "bits": 4, + "leg": 1, + "engine": "pair", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.44813395803794265, + "output_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 3.1054310826584697, + "output_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1219559424, + "peak_physical_footprint_bytes": 1633060544 + }, + "tag": "text-4-1-pair", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 290.3, + "per_stream_tps": 290.3, + "ttft_s_max": 0.01, + "timing_valid": true, + "end_to_end_tps": 287.21089724767234, + "makespan_s": 0.891331082675606, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 290.9520059688799, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8892927076667547, + "end_to_end_tps": 287.869222127852, + "timing_valid": true, + "per_stream_tps": [ + 291.0 + ], + "ttft_s": [ + 0.01 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 290.3169617686004, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.891331082675606, + "end_to_end_tps": 287.21089724767234, + "timing_valid": true, + "per_stream_tps": [ + 290.3 + ], + "ttft_s": [ + 0.01 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 289.59796729011634, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8934215418994427, + "end_to_end_tps": 286.53887106386065, + "timing_valid": true, + "per_stream_tps": [ + 289.6 + ], + "ttft_s": [ + 0.01 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd" + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 803.1, + "per_stream_tps": 203.8, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 797.7738681607349, + "makespan_s": 1.2835717499256134, + "steady_tps": 826.3, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 802.3079461677784, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2849068748764694, + "end_to_end_tps": 796.9449148588664, + "timing_valid": true, + "steady_tps": 825.52035392919, + "per_stream_tps": [ + 204.6, + 200.6, + 202.6, + 206.6 + ], + "ttft_s": [ + 0.04, + 0.01, + 0.03, + 0.05 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 803.075121796493, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2835717499256134, + "end_to_end_tps": 797.7738681607349, + "timing_valid": true, + "steady_tps": 826.3215944126978, + "per_stream_tps": [ + 206.8, + 204.8, + 202.8, + 200.8 + ], + "ttft_s": [ + 0.05, + 0.04, + 0.03, + 0.01 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 804.5218329148115, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2813565409742296, + "end_to_end_tps": 799.153059476671, + "timing_valid": true, + "steady_tps": 827.8513023551202, + "per_stream_tps": [ + 205.1, + 201.1, + 207.2, + 203.1 + ], + "ttft_s": [ + 0.04, + 0.01, + 0.05, + 0.03 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 303.1, + "per_stream_tps": 303.1, + "ttft_s_max": 0.01, + "timing_valid": true, + "end_to_end_tps": 299.72765481533753, + "makespan_s": 0.8541087079793215, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 303.1322775390812, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8541087079793215, + "end_to_end_tps": 299.72765481533753, + "timing_valid": true, + "per_stream_tps": [ + 303.1 + ], + "ttft_s": [ + 0.01 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "877a197cb9bca710" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 302.9378135524289, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8546056663617492, + "end_to_end_tps": 299.55336136472187, + "timing_valid": true, + "per_stream_tps": [ + 302.9 + ], + "ttft_s": [ + 0.01 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "877a197cb9bca710" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 303.77914667131824, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8522813338786364, + "end_to_end_tps": 300.3703000686086, + "timing_valid": true, + "per_stream_tps": [ + 303.8 + ], + "ttft_s": [ + 0.01 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "877a197cb9bca710" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 821.0, + "per_stream_tps": 208.4, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 815.381216015813, + "makespan_s": 1.255854292307049, + "steady_tps": 841.5, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 820.6586326979756, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2564638750627637, + "end_to_end_tps": 814.9856277792694, + "timing_valid": true, + "steady_tps": 841.0315240763456, + "per_stream_tps": [ + 207.2, + 209.3, + 211.5, + 205.2 + ], + "ttft_s": [ + 0.03, + 0.04, + 0.05, + 0.01 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "877a197cb9bca710", + "877a197cb9bca710", + "877a197cb9bca710", + "877a197cb9bca710" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 822.0500138473765, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2543224580585957, + "end_to_end_tps": 816.3769957407267, + "timing_valid": true, + "steady_tps": 842.4920241040868, + "per_stream_tps": [ + 207.6, + 205.5, + 211.8, + 209.7 + ], + "ttft_s": [ + 0.03, + 0.01, + 0.05, + 0.04 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "877a197cb9bca710", + "877a197cb9bca710", + "877a197cb9bca710", + "877a197cb9bca710" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 821.0467281767421, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.255854292307049, + "end_to_end_tps": 815.381216015813, + "timing_valid": true, + "steady_tps": 841.4699831140157, + "per_stream_tps": [ + 207.3, + 209.4, + 211.6, + 205.3 + ], + "ttft_s": [ + 0.03, + 0.04, + 0.05, + 0.01 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "877a197cb9bca710", + "877a197cb9bca710", + "877a197cb9bca710", + "877a197cb9bca710" + ] + } + ] + } + ] + }, + { + "family": "text", + "bits": 4, + "leg": 2, + "engine": "standard", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.15513708302751184, + "output_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "request_body_sha256": "584518b628ecabc5f4c40ba3e95794a33392a188f9fb876df4cb6a8b7dcb482f" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.3685434157960117, + "output_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "request_body_sha256": "93a7d790972a153b5d4a677f283d7fb40e6c1ef4848c059f958c0c28315c2311" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "unverified if server omits token hashes; text digest is not substituted" + }, + "memory": { + "max_rss_bytes": 1481539584, + "peak_physical_footprint_bytes": 2907719840 + }, + "tag": "text-4-2-standard", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 345.2, + "per_stream_tps": 345.2, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 338.2361054883265, + "makespan_s": 0.7568677496165037, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 345.19261736811393, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7568677496165037, + "end_to_end_tps": 338.2361054883265, + "timing_valid": true, + "per_stream_tps": [ + 345.2 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "05f89317f1e4d95f" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 345.2439289326284, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7566347080282867, + "end_to_end_tps": 338.3402813586361, + "timing_valid": true, + "per_stream_tps": [ + 345.2 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "05f89317f1e4d95f" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 344.38443484343003, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7586020827293396, + "end_to_end_tps": 337.46282251025383, + "timing_valid": true, + "per_stream_tps": [ + 344.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "05f89317f1e4d95f" + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 994.2, + "per_stream_tps": 252.8, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 967.5738736168812, + "makespan_s": 1.0583171248435974, + "steady_tps": 1016.2, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 991.7872480932048, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.0614519170485437, + "end_to_end_tps": 964.7163319911072, + "timing_valid": true, + "steady_tps": 1014.1923472850906, + "per_stream_tps": [ + 252.6, + 250.0, + 252.6, + 252.6 + ], + "ttft_s": [ + 0.05, + 0.03, + 0.05, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "05f89317f1e4d95f", + "05f89317f1e4d95f", + "05f89317f1e4d95f", + "05f89317f1e4d95f" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 994.2739082278363, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.0581944580189884, + "end_to_end_tps": 967.6860356243004, + "timing_valid": true, + "steady_tps": 1016.2892811091745, + "per_stream_tps": [ + 250.6, + 253.1, + 253.2, + 253.1 + ], + "ttft_s": [ + 0.03, + 0.05, + 0.05, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "05f89317f1e4d95f", + "05f89317f1e4d95f", + "05f89317f1e4d95f", + "05f89317f1e4d95f" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 994.2019508813183, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.0583171248435974, + "end_to_end_tps": 967.5738736168812, + "timing_valid": true, + "steady_tps": 1016.1900729339967, + "per_stream_tps": [ + 250.5, + 253.1, + 253.1, + 253.1 + ], + "ttft_s": [ + 0.03, + 0.05, + 0.05, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "05f89317f1e4d95f", + "05f89317f1e4d95f", + "05f89317f1e4d95f", + "05f89317f1e4d95f" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 346.5, + "per_stream_tps": 346.5, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 338.3110133849357, + "makespan_s": 0.7567001660354435, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 345.7885383036195, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7582044997252524, + "end_to_end_tps": 337.63977936396543, + "timing_valid": true, + "per_stream_tps": [ + 345.8 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "3e1007a53dac19be" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 346.64200758602084, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7566202501766384, + "end_to_end_tps": 338.3467465220961, + "timing_valid": true, + "per_stream_tps": [ + 346.6 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "3e1007a53dac19be" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 346.5114895390567, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7567001660354435, + "end_to_end_tps": 338.3110133849357, + "timing_valid": true, + "per_stream_tps": [ + 346.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "3e1007a53dac19be" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 1009.6, + "per_stream_tps": 254.5, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 970.1221257337911, + "makespan_s": 1.0555372079834342, + "steady_tps": 1020.8, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 1010.2358589462243, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.05518600018695, + "end_to_end_tps": 970.4450208954396, + "timing_valid": true, + "steady_tps": 1021.291667754826, + "per_stream_tps": [ + 254.7, + 254.7, + 254.6, + 254.7 + ], + "ttft_s": [ + 0.05, + 0.05, + 0.04, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "8951a9c3a9d80406", + "8951a9c3a9d80406", + "13646075c33d9905", + "8951a9c3a9d80406" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 1009.5637607136665, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.0555372079834342, + "end_to_end_tps": 970.1221257337911, + "timing_valid": true, + "steady_tps": 1020.7686133140712, + "per_stream_tps": [ + 254.5, + 254.5, + 254.5, + 254.4 + ], + "ttft_s": [ + 0.05, + 0.05, + 0.05, + 0.04 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "8951a9c3a9d80406", + "8951a9c3a9d80406", + "8951a9c3a9d80406", + "13646075c33d9905" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 1008.5710832176526, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.056892459280789, + "end_to_end_tps": 968.8781398788935, + "timing_valid": true, + "steady_tps": 1019.7883089503392, + "per_stream_tps": [ + 254.2, + 254.3, + 254.3, + 254.3 + ], + "ttft_s": [ + 0.04, + 0.05, + 0.05, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "13646075c33d9905", + "8951a9c3a9d80406", + "8951a9c3a9d80406", + "8951a9c3a9d80406" + ] + } + ] + } + ] + }, + { + "family": "text", + "bits": 4, + "leg": 3, + "engine": "standard", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.14434775011613965, + "output_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "request_body_sha256": "584518b628ecabc5f4c40ba3e95794a33392a188f9fb876df4cb6a8b7dcb482f" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.36950491601601243, + "output_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "request_body_sha256": "93a7d790972a153b5d4a677f283d7fb40e6c1ef4848c059f958c0c28315c2311" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "unverified if server omits token hashes; text digest is not substituted" + }, + "memory": { + "max_rss_bytes": 1478901760, + "peak_physical_footprint_bytes": 2903460072 + }, + "tag": "text-4-3-standard", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 343.2, + "per_stream_tps": 343.2, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 336.2598075599387, + "makespan_s": 0.7613160843029618, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 343.2046352618203, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7613160843029618, + "end_to_end_tps": 336.2598075599387, + "timing_valid": true, + "per_stream_tps": [ + 343.2 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "05f89317f1e4d95f" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 343.1096022772151, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7613788335584104, + "end_to_end_tps": 336.2320946112308, + "timing_valid": true, + "per_stream_tps": [ + 343.1 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "05f89317f1e4d95f" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 344.1117918905074, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7592371250502765, + "end_to_end_tps": 337.18056132074383, + "timing_valid": true, + "per_stream_tps": [ + 344.1 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "05f89317f1e4d95f" + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 995.8, + "per_stream_tps": 253.5, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 969.1229672696018, + "makespan_s": 1.0566254588775337, + "steady_tps": 1017.7, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 995.3185474232733, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.0579881658777595, + "end_to_end_tps": 967.8747201774595, + "timing_valid": true, + "steady_tps": 1017.6519863905525, + "per_stream_tps": [ + 250.8, + 253.5, + 253.5, + 253.5 + ], + "ttft_s": [ + 0.03, + 0.05, + 0.05, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "05f89317f1e4d95f", + "05f89317f1e4d95f", + "05f89317f1e4d95f", + "05f89317f1e4d95f" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 995.7835001775924, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.0566254588775337, + "end_to_end_tps": 969.1229672696018, + "timing_valid": true, + "steady_tps": 1017.6199878851983, + "per_stream_tps": [ + 253.5, + 253.5, + 250.9, + 253.5 + ], + "ttft_s": [ + 0.05, + 0.05, + 0.03, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "05f89317f1e4d95f", + "05f89317f1e4d95f", + "05f89317f1e4d95f", + "05f89317f1e4d95f" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 996.4038700852991, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.0554912500083447, + "end_to_end_tps": 970.1643665846632, + "timing_valid": true, + "steady_tps": 1018.0398765426199, + "per_stream_tps": [ + 251.1, + 253.6, + 253.6, + 253.6 + ], + "ttft_s": [ + 0.03, + 0.05, + 0.05, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "05f89317f1e4d95f", + "05f89317f1e4d95f", + "05f89317f1e4d95f", + "05f89317f1e4d95f" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 344.4, + "per_stream_tps": 344.4, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 336.31854359767317, + "makespan_s": 0.7611831249669194, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 344.3443437487388, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7614892502315342, + "end_to_end_tps": 336.18334063437146, + "timing_valid": true, + "per_stream_tps": [ + 344.3 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "3e1007a53dac19be" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 344.42151114972677, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.76113195810467, + "end_to_end_tps": 336.3411525085315, + "timing_valid": true, + "per_stream_tps": [ + 344.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "3e1007a53dac19be" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 344.39716757914834, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7611831249669194, + "end_to_end_tps": 336.31854359767317, + "timing_valid": true, + "per_stream_tps": [ + 344.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "3e1007a53dac19be" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 1007.4, + "per_stream_tps": 253.9, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 967.9249612607977, + "makespan_s": 1.057933249976486, + "steady_tps": 1018.5, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 1007.4954284061014, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.0578411254100502, + "end_to_end_tps": 968.0092552679569, + "timing_valid": true, + "steady_tps": 1018.6125681416439, + "per_stream_tps": [ + 254.0, + 254.0, + 254.0, + 253.9 + ], + "ttft_s": [ + 0.05, + 0.05, + 0.05, + 0.04 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "8951a9c3a9d80406", + "8951a9c3a9d80406", + "8951a9c3a9d80406", + "13646075c33d9905" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 1007.2399072151187, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.058043667115271, + "end_to_end_tps": 967.8239488847466, + "timing_valid": true, + "steady_tps": 1018.2805109629691, + "per_stream_tps": [ + 253.9, + 253.9, + 253.8, + 253.9 + ], + "ttft_s": [ + 0.05, + 0.05, + 0.04, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "8951a9c3a9d80406", + "8951a9c3a9d80406", + "13646075c33d9905", + "8951a9c3a9d80406" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 1007.376937033784, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.057933249976486, + "end_to_end_tps": 967.9249612607977, + "timing_valid": true, + "steady_tps": 1018.4917184438717, + "per_stream_tps": [ + 254.0, + 254.0, + 253.9, + 254.0 + ], + "ttft_s": [ + 0.05, + 0.05, + 0.04, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "8951a9c3a9d80406", + "8951a9c3a9d80406", + "13646075c33d9905", + "8951a9c3a9d80406" + ] + } + ] + } + ] + }, + { + "family": "text", + "bits": 4, + "leg": 4, + "engine": "pair", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.44820916606113315, + "output_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 3.0843422091566026, + "output_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1219608576, + "peak_physical_footprint_bytes": 1633175280 + }, + "tag": "text-4-4-pair", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 290.8, + "per_stream_tps": 290.8, + "ttft_s_max": 0.01, + "timing_valid": true, + "end_to_end_tps": 287.6940150570259, + "makespan_s": 0.8898342913016677, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 291.27296249058634, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8883634996600449, + "end_to_end_tps": 288.17032678398533, + "timing_valid": true, + "per_stream_tps": [ + 291.3 + ], + "ttft_s": [ + 0.01 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 290.5631499680331, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8905890001915395, + "end_to_end_tps": 287.45021546969696, + "timing_valid": true, + "per_stream_tps": [ + 290.6 + ], + "ttft_s": [ + 0.01 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 290.78887737569136, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8898342913016677, + "end_to_end_tps": 287.6940150570259, + "timing_valid": true, + "per_stream_tps": [ + 290.8 + ], + "ttft_s": [ + 0.01 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd" + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 804.0, + "per_stream_tps": 203.9, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 798.5724474615891, + "makespan_s": 1.2822881671600044, + "steady_tps": 827.2, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 803.3898590435937, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2831465830095112, + "end_to_end_tps": 798.0382082289422, + "timing_valid": true, + "steady_tps": 826.6476852518651, + "per_stream_tps": [ + 200.9, + 202.8, + 204.8, + 206.9 + ], + "ttft_s": [ + 0.01, + 0.03, + 0.04, + 0.05 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 803.9665769120127, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2822881671600044, + "end_to_end_tps": 798.5724474615891, + "timing_valid": true, + "steady_tps": 827.2259674479797, + "per_stream_tps": [ + 205.0, + 203.0, + 207.0, + 201.0 + ], + "ttft_s": [ + 0.04, + 0.03, + 0.05, + 0.01 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 804.4546280021725, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2813079580664635, + "end_to_end_tps": 799.1833606850067, + "timing_valid": true, + "steady_tps": 827.8567129294302, + "per_stream_tps": [ + 205.1, + 207.2, + 203.1, + 201.1 + ], + "ttft_s": [ + 0.04, + 0.05, + 0.03, + 0.01 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 303.8, + "per_stream_tps": 303.8, + "ttft_s_max": 0.01, + "timing_valid": true, + "end_to_end_tps": 300.3792875176562, + "makespan_s": 0.8522558333352208, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 303.4102870845408, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8532577091827989, + "end_to_end_tps": 300.0265889717915, + "timing_valid": true, + "per_stream_tps": [ + 303.4 + ], + "ttft_s": [ + 0.01 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "877a197cb9bca710" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 303.7890236542058, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8522558333352208, + "end_to_end_tps": 300.3792875176562, + "timing_valid": true, + "per_stream_tps": [ + 303.8 + ], + "ttft_s": [ + 0.01 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "877a197cb9bca710" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 304.5208963567468, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8502224576659501, + "end_to_end_tps": 301.09766884160763, + "timing_valid": true, + "per_stream_tps": [ + 304.5 + ], + "ttft_s": [ + 0.01 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "877a197cb9bca710" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 822.7, + "per_stream_tps": 208.8, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 816.991318314064, + "makespan_s": 1.2533792918547988, + "steady_tps": 843.3, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 822.7416822667194, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2532218750566244, + "end_to_end_tps": 817.0939403317809, + "timing_valid": true, + "steady_tps": 843.2798782572861, + "per_stream_tps": [ + 209.9, + 212.0, + 207.8, + 205.7 + ], + "ttft_s": [ + 0.04, + 0.05, + 0.03, + 0.01 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "877a197cb9bca710", + "877a197cb9bca710", + "877a197cb9bca710", + "877a197cb9bca710" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 822.5273302937994, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2535914997570217, + "end_to_end_tps": 816.8530180672711, + "timing_valid": true, + "steady_tps": 843.0682476896842, + "per_stream_tps": [ + 205.6, + 209.8, + 212.0, + 207.7 + ], + "ttft_s": [ + 0.01, + 0.04, + 0.05, + 0.03 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "877a197cb9bca710", + "877a197cb9bca710", + "877a197cb9bca710", + "877a197cb9bca710" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 822.7627254064802, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2533792918547988, + "end_to_end_tps": 816.991318314064, + "timing_valid": true, + "steady_tps": 843.315904803317, + "per_stream_tps": [ + 205.7, + 207.8, + 209.9, + 212.0 + ], + "ttft_s": [ + 0.01, + 0.03, + 0.04, + 0.05 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "877a197cb9bca710", + "877a197cb9bca710", + "877a197cb9bca710", + "877a197cb9bca710" + ] + } + ] + } + ] + }, + { + "family": "text", + "bits": 4, + "leg": 5, + "engine": "current", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.859181416220963, + "output_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.617500749882311, + "output_sha256": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1215660032, + "peak_physical_footprint_bytes": 1630979752 + }, + "tag": "text-4-5-current", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 315.0, + "per_stream_tps": 315.0, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 308.2580806430351, + "makespan_s": 0.8304729578085244, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 315.2135178299184, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8299119588918984, + "end_to_end_tps": 308.46645509460086, + "timing_valid": true, + "per_stream_tps": [ + 315.2 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 314.3251048476253, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8322252910584211, + "end_to_end_tps": 307.6090125480567, + "timing_valid": true, + "per_stream_tps": [ + 314.3 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 314.99632656653495, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8304729578085244, + "end_to_end_tps": 308.2580806430351, + "timing_valid": true, + "per_stream_tps": [ + 315.0 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd" + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 861.5, + "per_stream_tps": 221.2, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 849.650334720653, + "makespan_s": 1.2052016672678292, + "steady_tps": 907.8, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 861.1887251732501, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2060656249523163, + "end_to_end_tps": 849.0416929348148, + "timing_valid": true, + "steady_tps": 907.2366205240755, + "per_stream_tps": [ + 215.3, + 227.1, + 223.0, + 219.1 + ], + "ttft_s": [ + 0.02, + 0.08, + 0.06, + 0.04 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 862.3178205332723, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2045725001953542, + "end_to_end_tps": 850.0941203903712, + "timing_valid": true, + "steady_tps": 908.4156477720528, + "per_stream_tps": [ + 219.4, + 215.6, + 227.4, + 223.3 + ], + "ttft_s": [ + 0.04, + 0.02, + 0.08, + 0.06 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 861.4812410115998, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2052016672678292, + "end_to_end_tps": 849.650334720653, + "timing_valid": true, + "steady_tps": 907.836756377548, + "per_stream_tps": [ + 223.2, + 219.2, + 227.1, + 215.5 + ], + "ttft_s": [ + 0.06, + 0.04, + 0.08, + 0.02 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd", + "8b9be652dd2c88bd" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 328.4, + "per_stream_tps": 328.4, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 320.8534200822005, + "makespan_s": 0.7978721247054636, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 328.39192909520665, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7978721247054636, + "end_to_end_tps": 320.8534200822005, + "timing_valid": true, + "per_stream_tps": [ + 328.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "555fe37e25c90f95" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 327.92201272767653, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7989699593745172, + "end_to_end_tps": 320.41254742595396, + "timing_valid": true, + "per_stream_tps": [ + 327.9 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "555fe37e25c90f95" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 328.708586074887, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7971310000866652, + "end_to_end_tps": 321.1517303582063, + "timing_valid": true, + "per_stream_tps": [ + 328.7 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "555fe37e25c90f95" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 880.1, + "per_stream_tps": 226.1, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 867.0230251399094, + "makespan_s": 1.181052832864225, + "steady_tps": 929.3, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 880.096931462885, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1806820831261575, + "end_to_end_tps": 867.2952817990583, + "timing_valid": true, + "steady_tps": 929.6553204916108, + "per_stream_tps": [ + 232.6, + 224.1, + 228.3, + 220.1 + ], + "ttft_s": [ + 0.08, + 0.04, + 0.06, + 0.02 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "555fe37e25c90f95", + "555fe37e25c90f95", + "555fe37e25c90f95", + "555fe37e25c90f95" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 880.0811742962081, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.181052832864225, + "end_to_end_tps": 867.0230251399094, + "timing_valid": true, + "steady_tps": 929.3417831463714, + "per_stream_tps": [ + 232.6, + 220.0, + 224.1, + 228.2 + ], + "ttft_s": [ + 0.08, + 0.02, + 0.04, + 0.06 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "555fe37e25c90f95", + "555fe37e25c90f95", + "555fe37e25c90f95", + "555fe37e25c90f95" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 879.6394512331002, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.181590875145048, + "end_to_end_tps": 866.628222627648, + "timing_valid": true, + "steady_tps": 928.9763973819266, + "per_stream_tps": [ + 223.9, + 219.9, + 232.5, + 228.1 + ], + "ttft_s": [ + 0.04, + 0.02, + 0.08, + 0.06 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "555fe37e25c90f95", + "555fe37e25c90f95", + "555fe37e25c90f95", + "555fe37e25c90f95" + ] + } + ] + } + ] + }, + { + "family": "vl", + "bits": 4, + "leg": 0, + "engine": "current", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.8617584998719394, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.650635291822255, + "output_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1833582592, + "peak_physical_footprint_bytes": 2290124912 + }, + "tag": "vl-4-0-current", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 299.0, + "per_stream_tps": 299.0, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 292.8892693414407, + "makespan_s": 0.8740504579618573, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 298.9678136235486, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8740504579618573, + "end_to_end_tps": 292.8892693414407, + "timing_valid": true, + "per_stream_tps": [ + 299.0 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "62f3c8997790ce94" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 299.77093980533385, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8717442909255624, + "end_to_end_tps": 293.6640969890328, + "timing_valid": true, + "per_stream_tps": [ + 299.8 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "62f3c8997790ce94" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 298.1739624282216, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8762280000373721, + "end_to_end_tps": 292.1614009014564, + "timing_valid": true, + "per_stream_tps": [ + 298.2 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "62f3c8997790ce94" + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 835.3, + "per_stream_tps": 214.3, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 823.7748606792014, + "makespan_s": 1.2430580840446055, + "steady_tps": 877.0, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 835.9654462267201, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2419054172933102, + "end_to_end_tps": 824.5394421676431, + "timing_valid": true, + "steady_tps": 877.7972021739898, + "per_stream_tps": [ + 216.3, + 212.6, + 220.1, + 209.0 + ], + "ttft_s": [ + 0.06, + 0.04, + 0.08, + 0.02 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "62f3c8997790ce94", + "62f3c8997790ce94", + "62f3c8997790ce94", + "62f3c8997790ce94" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 835.262973515151, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2430580840446055, + "end_to_end_tps": 823.7748606792014, + "timing_valid": true, + "steady_tps": 877.0020009028603, + "per_stream_tps": [ + 219.9, + 208.8, + 212.4, + 216.1 + ], + "ttft_s": [ + 0.08, + 0.02, + 0.04, + 0.06 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "62f3c8997790ce94", + "62f3c8997790ce94", + "62f3c8997790ce94", + "62f3c8997790ce94" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 834.8256810049306, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2433419581502676, + "end_to_end_tps": 823.586780199564, + "timing_valid": true, + "steady_tps": 876.9013782612393, + "per_stream_tps": [ + 219.8, + 216.1, + 212.4, + 208.8 + ], + "ttft_s": [ + 0.08, + 0.06, + 0.04, + 0.02 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "62f3c8997790ce94", + "62f3c8997790ce94", + "62f3c8997790ce94", + "62f3c8997790ce94" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 295.9, + "per_stream_tps": 295.9, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 289.886527378659, + "makespan_s": 0.8831041660159826, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 295.7815605988286, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8835677918978035, + "end_to_end_tps": 289.73441805765805, + "timing_valid": true, + "per_stream_tps": [ + 295.8 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "9cbca803b391f42b" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 296.3946782287728, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8817843748256564, + "end_to_end_tps": 290.3204086039918, + "timing_valid": true, + "per_stream_tps": [ + 296.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "9cbca803b391f42b" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 295.92509717112875, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8831041660159826, + "end_to_end_tps": 289.886527378659, + "timing_valid": true, + "per_stream_tps": [ + 295.9 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "9cbca803b391f42b" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 831.5, + "per_stream_tps": 213.4, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 820.042347803922, + "makespan_s": 1.2487159995362163, + "steady_tps": 874.4, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 830.6755448472516, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.24992208275944, + "end_to_end_tps": 819.2510670259749, + "timing_valid": true, + "steady_tps": 873.236949221183, + "per_stream_tps": [ + 215.0, + 211.3, + 218.8, + 207.7 + ], + "ttft_s": [ + 0.06, + 0.04, + 0.08, + 0.02 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "9cbca803b391f42b", + "9cbca803b391f42b", + "9cbca803b391f42b", + "9cbca803b391f42b" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 832.676823463046, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2468495001085103, + "end_to_end_tps": 821.2699286568939, + "timing_valid": true, + "steady_tps": 875.6201251547575, + "per_stream_tps": [ + 215.6, + 211.8, + 219.4, + 208.2 + ], + "ttft_s": [ + 0.06, + 0.04, + 0.08, + 0.02 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "9cbca803b391f42b", + "9cbca803b391f42b", + "9cbca803b391f42b", + "9cbca803b391f42b" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 831.500584634071, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2487159995362163, + "end_to_end_tps": 820.042347803922, + "timing_valid": true, + "steady_tps": 874.3601159862544, + "per_stream_tps": [ + 215.3, + 219.1, + 207.9, + 211.5 + ], + "ttft_s": [ + 0.06, + 0.08, + 0.02, + 0.04 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "9cbca803b391f42b", + "9cbca803b391f42b", + "9cbca803b391f42b", + "9cbca803b391f42b" + ] + } + ] + } + ] + }, + { + "family": "vl", + "bits": 4, + "leg": 1, + "engine": "standard", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.1587639581412077, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "dc74777df7a1af0d8499ba22a5e658bb688258791b9fc3122648802f2a15b7a9" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.3743875832296908, + "output_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "request_body_sha256": "6c63b60f0c2f655893c1dc82a189d8655d35848bc94c4a652454f33c042426e2" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "unverified if server omits token hashes; text digest is not substituted" + }, + "memory": { + "max_rss_bytes": 2099281920, + "peak_physical_footprint_bytes": 3430222656 + }, + "tag": "vl-4-1-standard", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 325.4, + "per_stream_tps": 325.4, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 319.0181057477032, + "makespan_s": 0.8024622909724712, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 324.52480041957267, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8041047500446439, + "end_to_end_tps": 318.3664814637482, + "timing_valid": true, + "per_stream_tps": [ + 324.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "2b4e83287ef4a0ce" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 325.4062697216241, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8024622909724712, + "end_to_end_tps": 319.0181057477032, + "timing_valid": true, + "per_stream_tps": [ + 325.4 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "2b4e83287ef4a0ce" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 325.548105694293, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8018392501398921, + "end_to_end_tps": 319.2659874848197, + "timing_valid": true, + "per_stream_tps": [ + 325.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "2b4e83287ef4a0ce" + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 948.4, + "per_stream_tps": 240.2, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 923.8158304606382, + "makespan_s": 1.108446041122079, + "steady_tps": 962.4, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 936.2312188950415, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1232852078974247, + "end_to_end_tps": 911.6117552342138, + "timing_valid": true, + "steady_tps": 950.817650319079, + "per_stream_tps": [ + 238.4, + 238.4, + 238.4, + 236.0 + ], + "ttft_s": [ + 0.05, + 0.05, + 0.05, + 0.03 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "77900a1d84aff413", + "77900a1d84aff413", + "77900a1d84aff413", + "77900a1d84aff413" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 948.4289805455426, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.108446041122079, + "end_to_end_tps": 923.8158304606382, + "timing_valid": true, + "steady_tps": 962.3991525881518, + "per_stream_tps": [ + 241.3, + 241.3, + 239.1, + 241.3 + ], + "ttft_s": [ + 0.05, + 0.05, + 0.03, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "77900a1d84aff413", + "77900a1d84aff413", + "77900a1d84aff413", + "77900a1d84aff413" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 948.6774045902526, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1079975003376603, + "end_to_end_tps": 924.1898106159425, + "timing_valid": true, + "steady_tps": 962.4598146789826, + "per_stream_tps": [ + 241.4, + 239.1, + 241.4, + 241.4 + ], + "ttft_s": [ + 0.05, + 0.03, + 0.05, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "77900a1d84aff413", + "77900a1d84aff413", + "77900a1d84aff413", + "77900a1d84aff413" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 324.3, + "per_stream_tps": 324.3, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 318.1617375609637, + "makespan_s": 0.804622208699584, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 324.32322412256036, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8048312501050532, + "end_to_end_tps": 318.0791003910258, + "timing_valid": true, + "per_stream_tps": [ + 324.3 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "430c3620abbb5db4" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 324.2902107474412, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.804622208699584, + "end_to_end_tps": 318.1617375609637, + "timing_valid": true, + "per_stream_tps": [ + 324.3 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "430c3620abbb5db4" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 324.6449777585084, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8040862078778446, + "end_to_end_tps": 318.3738229705976, + "timing_valid": true, + "per_stream_tps": [ + 324.6 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "430c3620abbb5db4" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 946.7, + "per_stream_tps": 239.8, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 922.1124872288405, + "makespan_s": 1.1104935831390321, + "steady_tps": 959.4, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 937.3603934366955, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1213881252333522, + "end_to_end_tps": 913.1539535313998, + "timing_valid": true, + "steady_tps": 950.146192777668, + "per_stream_tps": [ + 238.5, + 236.3, + 238.5, + 238.5 + ], + "ttft_s": [ + 0.05, + 0.03, + 0.05, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "9c50367eb37de1e7", + "98395f661f4091e6", + "9c50367eb37de1e7", + "9c50367eb37de1e7" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 946.7368475680894, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1104253749363124, + "end_to_end_tps": 922.1691282574758, + "timing_valid": true, + "steady_tps": 959.4432886545605, + "per_stream_tps": [ + 238.6, + 240.9, + 240.9, + 240.9 + ], + "ttft_s": [ + 0.03, + 0.05, + 0.05, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "98395f661f4091e6", + "9c50367eb37de1e7", + "9c50367eb37de1e7", + "9c50367eb37de1e7" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 946.7620750077942, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1104935831390321, + "end_to_end_tps": 922.1124872288405, + "timing_valid": true, + "steady_tps": 959.350582828662, + "per_stream_tps": [ + 238.6, + 240.9, + 240.9, + 240.9 + ], + "ttft_s": [ + 0.03, + 0.05, + 0.05, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "98395f661f4091e6", + "9c50367eb37de1e7", + "9c50367eb37de1e7", + "9c50367eb37de1e7" + ] + } + ] + } + ] + }, + { + "family": "vl", + "bits": 4, + "leg": 2, + "engine": "standard", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.15947616565972567, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "dc74777df7a1af0d8499ba22a5e658bb688258791b9fc3122648802f2a15b7a9" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.373862124979496, + "output_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "request_body_sha256": "6c63b60f0c2f655893c1dc82a189d8655d35848bc94c4a652454f33c042426e2" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "unverified if server omits token hashes; text digest is not substituted" + }, + "memory": { + "max_rss_bytes": 2099347456, + "peak_physical_footprint_bytes": 3431058000 + }, + "tag": "vl-4-2-standard", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 326.2, + "per_stream_tps": 326.2, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 319.9500913911257, + "makespan_s": 0.8001247909851372, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 326.25782132840624, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.800016833934933, + "end_to_end_tps": 319.9932665677145, + "timing_valid": true, + "per_stream_tps": [ + 326.3 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "2b4e83287ef4a0ce" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 326.1896899753963, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8001247909851372, + "end_to_end_tps": 319.9500913911257, + "timing_valid": true, + "per_stream_tps": [ + 326.2 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "2b4e83287ef4a0ce" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 325.25874965409633, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8025083746761084, + "end_to_end_tps": 318.99978626805154, + "timing_valid": true, + "per_stream_tps": [ + 325.3 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "2b4e83287ef4a0ce" + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 947.1, + "per_stream_tps": 240.1, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 922.1960154214537, + "makespan_s": 1.1103929998353124, + "steady_tps": 961.1, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 938.3710618056379, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1210402501747012, + "end_to_end_tps": 913.4373184552663, + "timing_valid": true, + "steady_tps": 953.3632075065676, + "per_stream_tps": [ + 239.0, + 236.6, + 239.0, + 239.1 + ], + "ttft_s": [ + 0.05, + 0.03, + 0.05, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "77900a1d84aff413", + "77900a1d84aff413", + "77900a1d84aff413", + "77900a1d84aff413" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 947.1276808380298, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1103929998353124, + "end_to_end_tps": 922.1960154214537, + "timing_valid": true, + "steady_tps": 961.1417523017727, + "per_stream_tps": [ + 238.8, + 241.0, + 241.0, + 241.0 + ], + "ttft_s": [ + 0.03, + 0.05, + 0.05, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "77900a1d84aff413", + "77900a1d84aff413", + "77900a1d84aff413", + "77900a1d84aff413" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 947.2582618985884, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1103473748080432, + "end_to_end_tps": 922.2339091647143, + "timing_valid": true, + "steady_tps": 961.2324522194795, + "per_stream_tps": [ + 241.0, + 241.0, + 238.8, + 241.1 + ], + "ttft_s": [ + 0.05, + 0.05, + 0.03, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "77900a1d84aff413", + "77900a1d84aff413", + "77900a1d84aff413", + "77900a1d84aff413" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 326.2, + "per_stream_tps": 326.2, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 319.98041798057204, + "makespan_s": 0.8000489580444992, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 325.10778768998364, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.802861999720335, + "end_to_end_tps": 318.85928103356963, + "timing_valid": true, + "per_stream_tps": [ + 325.1 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "430c3620abbb5db4" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 326.23361191631614, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8000489580444992, + "end_to_end_tps": 319.98041798057204, + "timing_valid": true, + "per_stream_tps": [ + 326.2 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "430c3620abbb5db4" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 326.49126088752274, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.7994025000371039, + "end_to_end_tps": 320.2391786216804, + "timing_valid": true, + "per_stream_tps": [ + 326.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + null + ], + "evidence_refs": [ + "430c3620abbb5db4" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 949.7, + "per_stream_tps": 240.8, + "ttft_s_max": 0.05, + "timing_valid": true, + "end_to_end_tps": 925.0079949362224, + "makespan_s": 1.1070174588821828, + "steady_tps": 962.5, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 942.5165398121123, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1153343748301268, + "end_to_end_tps": 918.1103201951992, + "timing_valid": true, + "steady_tps": 955.3339355792946, + "per_stream_tps": [ + 239.9, + 239.9, + 239.9, + 237.5 + ], + "ttft_s": [ + 0.05, + 0.05, + 0.05, + 0.03 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "9c50367eb37de1e7", + "9c50367eb37de1e7", + "9c50367eb37de1e7", + "98395f661f4091e6" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 949.6675788548038, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1068007498979568, + "end_to_end_tps": 925.1891093265064, + "timing_valid": true, + "steady_tps": 962.5047187300808, + "per_stream_tps": [ + 241.6, + 241.6, + 241.6, + 239.4 + ], + "ttft_s": [ + 0.05, + 0.05, + 0.05, + 0.03 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "9c50367eb37de1e7", + "9c50367eb37de1e7", + "9c50367eb37de1e7", + "98395f661f4091e6" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 949.7362559624695, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.1070174588821828, + "end_to_end_tps": 925.0079949362224, + "timing_valid": true, + "steady_tps": 962.6227955738335, + "per_stream_tps": [ + 241.6, + 241.6, + 239.4, + 241.6 + ], + "ttft_s": [ + 0.05, + 0.05, + 0.03, + 0.05 + ], + "errors": [], + "equal_alone": [ + null, + null, + null, + null + ], + "evidence_refs": [ + "9c50367eb37de1e7", + "9c50367eb37de1e7", + "98395f661f4091e6", + "9c50367eb37de1e7" + ] + } + ] + } + ] + }, + { + "family": "vl", + "bits": 4, + "leg": 3, + "engine": "current", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.8619535421021283, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.654370457865298, + "output_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1833598976, + "peak_physical_footprint_bytes": 2290223288 + }, + "tag": "vl-4-3-current", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 298.8, + "per_stream_tps": 298.8, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 292.7441617365916, + "makespan_s": 0.8744837078265846, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 299.82642413314005, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8715500421822071, + "end_to_end_tps": 293.72954805787316, + "timing_valid": true, + "per_stream_tps": [ + 299.8 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "62f3c8997790ce94" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 298.7991056159947, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8744837078265846, + "end_to_end_tps": 292.7441617365916, + "timing_valid": true, + "per_stream_tps": [ + 298.8 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "62f3c8997790ce94" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 298.7266044486912, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8746907082386315, + "end_to_end_tps": 292.674882205515, + "timing_valid": true, + "per_stream_tps": [ + 298.7 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "62f3c8997790ce94" + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 836.0, + "per_stream_tps": 214.5, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 824.7360678976339, + "makespan_s": 1.2416093340143561, + "steady_tps": 878.0, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 835.959765100696, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2419295418076217, + "end_to_end_tps": 824.523425467095, + "timing_valid": true, + "steady_tps": 877.7798046698534, + "per_stream_tps": [ + 209.0, + 216.3, + 220.1, + 212.6 + ], + "ttft_s": [ + 0.02, + 0.06, + 0.08, + 0.04 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "62f3c8997790ce94", + "62f3c8997790ce94", + "62f3c8997790ce94", + "62f3c8997790ce94" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 836.3024118586642, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2414488750509918, + "end_to_end_tps": 824.8426661612947, + "timing_valid": true, + "steady_tps": 878.1019975748836, + "per_stream_tps": [ + 212.7, + 209.1, + 220.2, + 216.4 + ], + "ttft_s": [ + 0.04, + 0.02, + 0.08, + 0.06 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "62f3c8997790ce94", + "62f3c8997790ce94", + "62f3c8997790ce94", + "62f3c8997790ce94" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 836.0459008171667, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2416093340143561, + "end_to_end_tps": 824.7360678976339, + "timing_valid": true, + "steady_tps": 878.0232994004378, + "per_stream_tps": [ + 216.4, + 209.1, + 212.6, + 220.1 + ], + "ttft_s": [ + 0.06, + 0.02, + 0.04, + 0.08 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "62f3c8997790ce94", + "62f3c8997790ce94", + "62f3c8997790ce94", + "62f3c8997790ce94" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 296.5, + "per_stream_tps": 296.5, + "ttft_s_max": 0.02, + "timing_valid": true, + "end_to_end_tps": 290.4599095196861, + "makespan_s": 0.8813608749769628, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 296.20792867098766, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8822562079876661, + "end_to_end_tps": 290.16514441299216, + "timing_valid": true, + "per_stream_tps": [ + 296.2 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "9cbca803b391f42b" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 296.6010393481105, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8811197499744594, + "end_to_end_tps": 290.5393960439776, + "timing_valid": true, + "per_stream_tps": [ + 296.6 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "9cbca803b391f42b" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 296.53415547090634, + "requested_requests": 1, + "timed_requests": 1, + "makespan_s": 0.8813608749769628, + "end_to_end_tps": 290.4599095196861, + "timing_valid": true, + "per_stream_tps": [ + 296.5 + ], + "ttft_s": [ + 0.02 + ], + "errors": [], + "equal_alone": [ + true + ], + "evidence_refs": [ + "9cbca803b391f42b" + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 831.1, + "per_stream_tps": 213.2, + "ttft_s_max": 0.08, + "timing_valid": true, + "end_to_end_tps": 819.5960562558075, + "makespan_s": 1.249395958147943, + "steady_tps": 873.7, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 831.0323498710067, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.249622625298798, + "end_to_end_tps": 819.4473909714549, + "timing_valid": true, + "steady_tps": 873.6588267335163, + "per_stream_tps": [ + 211.4, + 215.1, + 207.8, + 218.9 + ], + "ttft_s": [ + 0.04, + 0.06, + 0.02, + 0.08 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "9cbca803b391f42b", + "9cbca803b391f42b", + "9cbca803b391f42b", + "9cbca803b391f42b" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 831.1183756420012, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.249395958147943, + "end_to_end_tps": 819.5960562558075, + "timing_valid": true, + "steady_tps": 873.7252193805314, + "per_stream_tps": [ + 211.4, + 207.8, + 219.0, + 215.1 + ], + "ttft_s": [ + 0.04, + 0.02, + 0.08, + 0.06 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "9cbca803b391f42b", + "9cbca803b391f42b", + "9cbca803b391f42b", + "9cbca803b391f42b" + ] + }, + { + "failed": 0, + "unmeasured": 0, + "aggregate_tps": 831.2861329810726, + "requested_requests": 4, + "timed_requests": 4, + "makespan_s": 1.2491025417111814, + "end_to_end_tps": 819.7885808455669, + "timing_valid": true, + "steady_tps": 873.898657035953, + "per_stream_tps": [ + 219.0, + 211.4, + 207.8, + 215.1 + ], + "ttft_s": [ + 0.08, + 0.04, + 0.02, + 0.06 + ], + "errors": [], + "equal_alone": [ + true, + true, + true, + true + ], + "evidence_refs": [ + "9cbca803b391f42b", + "9cbca803b391f42b", + "9cbca803b391f42b", + "9cbca803b391f42b" + ] + } + ] + } + ] + } + ], + "evidence_pool": { + "8b9be652dd2c88bd": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "7ffd540d4c86", + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7", + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + "555fe37e25c90f95": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "97f95aa70ea3", + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a", + "output_sha256": "4ccfefec5a1d93de3915d5dd2a3ea41bdf4249072271c4474973ee62cd41d5db" + }, + "877a197cb9bca710": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "760b6cafb707", + "token_sha256": "760b6cafb70745f4d5f53bf8162dbd7ab4ef731fbc4e86ec4085e22e7e341880", + "output_sha256": "e71bf7a65f320c274e70c8c3dc0e56603ca48305ada0390153f7f17b781f1a3a" + }, + "05f89317f1e4d95f": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "4165638deeb903f6909295759dedff111c6f8a27a813180d00417b654d560e27" + }, + "3e1007a53dac19be": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "fc6c1799b10d5ece11495fb9eb854a5562db013147b5382e1e92d4dc95bf5ae7" + }, + "8951a9c3a9d80406": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f3e436a1e2277cc5f0ba02cc9de357c6286ea1a8d1c78cb277ec1629464af7b" + }, + "13646075c33d9905": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "24f4feefb469f86e1a3fc752cbd6d2404c1530c3c8f87fbf0408b1741da5017a" + }, + "62f3c8997790ce94": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "cd53eb1c94a6", + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee", + "output_sha256": "39535e4a52ca53e38c770d56e69e1281e9912138f6cc6d2ff299fa62c9352d41" + }, + "9cbca803b391f42b": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "a6dac318d393", + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa", + "output_sha256": "19c251e175f9bdc5934af69182c9c261c9843cebaffa6cca06f47b07853d553a" + }, + "2b4e83287ef4a0ce": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8f09aba86abc10a48c2aa0da4db383e8a734db540e2ac890f07b5e39578e9502" + }, + "77900a1d84aff413": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9d055e80eba3b89892354174f854afda62cea6fa3621c06ecbc986ae1517ef47" + }, + "430c3620abbb5db4": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "8c84c68381968d2a7264db9bdcd270d793ac6ecb227c7272d13158931f57cf17" + }, + "9c50367eb37de1e7": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "fa2d3eed4c2582f3077ad00bb99b4e2db17605f9bcc8dab56f3fa87c696c08e6" + }, + "98395f661f4091e6": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "372fd48a132419570d44767f9832f28ab8e663fbd6d31e8a2db438ef0ca86530" + } + } + }, + "limitations": [ + "M5 only,4bit text and VL,2k/8k cold prompts, solo/four streams; no6/8bit or32k/64k performance claims.", + "VL requests here contain text only; image correctness and image throughput are separate evidence.", + "Only paired TEXT4 candidate was measured. Both actual paired legs report gs64_tiles=true,twin=true,gs32_affine=false. No decode-twin fallback occurred; no GS32 VL candidate was enabled.", + "Standard omits token-id hashes: its alone/serial token equality is unverified. Text output digests are retained but do not replace token equality. Native hash checks passed.", + "Both request budgets are70000; native reserved caches and standard lazy KV/sliding caches allocate differently. Standard checkpoint caps remain text262144/VL128000; no model config edit or memory/wired limit override.", + "Peak physical footprint and maximum RSS are actual /usr/bin/time -l process measurements, not inferred GPU allocator/pinned-memory peaks.", + "Paired candidate versus current public is a full projection-path comparison, including tile/twin arithmetic; this screen does not isolate pair-wrapper-only cost.", + "M4 pair-only gains are retained separately and are not extrapolated to M5. This screen shows lower M5 decode throughput despite better paired cold prefill versus current.", + "Ten mirrored legs expose order variation but are not a thermal/frequency/bandwidth study; do not assign unmeasured hardware causes.", + "Correctness prerequisites do not establish an accepted whole-model fidelity/performance gate or full-backend qualification.", + "Fresh remote environment is separately frozen; transitive dependencies are not asserted identical to earlier or local M4 environments." + ], + "raw_file_manifest": { + "http-results/events.json": { + "bytes": 29001, + "sha256": "5d94aab893aa6c34a635f1025404ec02654a81e890d86f58846c86cfb0b053e9" + }, + "http-results/manifest.json": { + "bytes": 13718, + "sha256": "2979ff660ee2ba66acaafbbca056a8904eb2e50648a78c4eabd33ccff2b0de55" + }, + "http-results/records.json": { + "bytes": 10615, + "sha256": "33492b85b7e4f1cf88c3ea8ac03f559c094fd73e05831bb93f631c0f69fbbf54" + }, + "http-results/summary.json": { + "bytes": 5889, + "sha256": "332539f66df431b47a9455cecd4a935c0349ef55a949ade6a9c9bdac806e3be5" + }, + "http-results/text-4-0-current.after.health": { + "bytes": 303, + "sha256": "1eb7e69a913a82f6ebdd17f63480c74999d649050645ae34c7f22e35158c294b" + }, + "http-results/text-4-0-current.after.metrics": { + "bytes": 12306, + "sha256": "4a06dfa8b08ab316e9aadce390c57f2647fee84ee92d2a69a33d317434bf61fa" + }, + "http-results/text-4-0-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/text-4-0-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "http-results/text-4-0-current.before.metrics": { + "bytes": 12128, + "sha256": "1e218089fa310167b9ff2a239a738ea7a906d509c583e1fe7e238d2ae8744fe1" + }, + "http-results/text-4-0-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/text-4-0-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "67e69d3a4b0bf1937c21ee32296f5afbb692ecb6666a1aa6e62c52781246925e" + }, + "http-results/text-4-0-current.cold-2048.json": { + "bytes": 20296, + "sha256": "476c961a2df9848be6151d0c562dc08a7c209064747c6f95f34fabd97da03c3b" + }, + "http-results/text-4-0-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "d02f9dac9726aa6aadf83468fa066e48f81a085ef6cc82889b8ad6c134a7554e" + }, + "http-results/text-4-0-current.cold-8192.json": { + "bytes": 57146, + "sha256": "66575cf391fa68ee6b75d5334ca2f457049f83c76ebe5d70dfe1822147e39953" + }, + "http-results/text-4-0-current.concurrent.client.log": { + "bytes": 3239, + "sha256": "f1ea2d553703b4a60da1cbe5ae22c75329cb2b9c179da3a78de681ecbd4b3a52" + }, + "http-results/text-4-0-current.concurrent.json": { + "bytes": 282943, + "sha256": "b72083d1657819e9ab7892bc8cdd7509136dffc521d4b259d445e9cee6134d4b" + }, + "http-results/text-4-0-current.server.log": { + "bytes": 13669, + "sha256": "f5e987c4227f77c6885c14851a2ce47a7a527d6bf7c3660a5c51737d7fe4e0a5" + }, + "http-results/text-4-1-pair.after.health": { + "bytes": 303, + "sha256": "acb548a8d66f97bae9d59037d63d996d818be60c5414283d938bcbab52d14bda" + }, + "http-results/text-4-1-pair.after.metrics": { + "bytes": 12306, + "sha256": "14a55da48d4fc832418b82676457289ab064500abd4de69720b9ba3db725068f" + }, + "http-results/text-4-1-pair.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/text-4-1-pair.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "http-results/text-4-1-pair.before.metrics": { + "bytes": 12128, + "sha256": "bf58eedbdd42df2243693092393a36c9047aa8ce59034c22c79cb6d3f5c66361" + }, + "http-results/text-4-1-pair.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/text-4-1-pair.cold-2048.client.log": { + "bytes": 350, + "sha256": "00153e731229db7d2fb1f720c8361d6620269ad05b765d37b0c3b9183b9a68f0" + }, + "http-results/text-4-1-pair.cold-2048.json": { + "bytes": 20297, + "sha256": "afe080020fda797399fd28f96bd6ee1debb318795130922a07ca26b71e28e130" + }, + "http-results/text-4-1-pair.cold-8192.client.log": { + "bytes": 348, + "sha256": "6f2b529061d69f295347b36873821bfdfd297e8a69cfbd1a602c371e10c97f94" + }, + "http-results/text-4-1-pair.cold-8192.json": { + "bytes": 57144, + "sha256": "69caa7fb69eb933af9c369d1d67c5aed02ed589616c4d9dd3a4e08c798496544" + }, + "http-results/text-4-1-pair.concurrent.client.log": { + "bytes": 3219, + "sha256": "5352d578c2810e60335864fb249e35d8c81b3acdb4aa74137ed5edacab7829e0" + }, + "http-results/text-4-1-pair.concurrent.json": { + "bytes": 282936, + "sha256": "611e0176cebbc07a0d405ab17d00db5b76ee98b647dbd75d0a546421ee2e7b1e" + }, + "http-results/text-4-1-pair.server.log": { + "bytes": 13745, + "sha256": "ccf6078e0f3167f216f2cf23ab55175debdcbb9532fa5749d332b1d2e0f4776b" + }, + "http-results/text-4-2-standard.after.health": { + "bytes": 453, + "sha256": "4b9a34a2eec3165a5c1ef2da3e32af6676938546b56f6c54a64d8d3679bb3540" + }, + "http-results/text-4-2-standard.after.metrics": { + "bytes": 32122, + "sha256": "0e9628073815f37ecba37019f3d2654e7d85ddf68e10bc283a5a786ec09a166d" + }, + "http-results/text-4-2-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "http-results/text-4-2-standard.before.health": { + "bytes": 453, + "sha256": "4b9a34a2eec3165a5c1ef2da3e32af6676938546b56f6c54a64d8d3679bb3540" + }, + "http-results/text-4-2-standard.before.metrics": { + "bytes": 906, + "sha256": "310306af9bef83a55b666256c62c47dbb80c1dd72adc98baca161bc23c348b55" + }, + "http-results/text-4-2-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "http-results/text-4-2-standard.cold-2048.client.log": { + "bytes": 297, + "sha256": "184d6bfd63d825996968201776e52838f7318ccd13ce1a8f2d530e52ac544347" + }, + "http-results/text-4-2-standard.cold-2048.json": { + "bytes": 26076, + "sha256": "2f0e25001ad0a3dfbf93095ed6d398dccbbdd3c61f5950eb197ce3fa4b86b54e" + }, + "http-results/text-4-2-standard.cold-8192.client.log": { + "bytes": 295, + "sha256": "8700ccdd36b9fa65ea90349f0245442e8b9b693560a3081b03ae5ba84dc57363" + }, + "http-results/text-4-2-standard.cold-8192.json": { + "bytes": 62933, + "sha256": "d57c23dc1377da0492d26e5fcf14087c996c0678427302d47302642aac6e2014" + }, + "http-results/text-4-2-standard.concurrent.client.log": { + "bytes": 3236, + "sha256": "f970410e8dd1a8a960111bd7d4b03a3326b50998076204f2e96146e96c74450f" + }, + "http-results/text-4-2-standard.concurrent.json": { + "bytes": 285704, + "sha256": "de508307d2994f7383fac843fc8f951b47a21854036b670af3fa89f2d8ae92ba" + }, + "http-results/text-4-2-standard.server.log": { + "bytes": 50891, + "sha256": "b08e004c53995bd98a69c40d0dbcf71f0f5f5cb2a1590a2af211a99b31d6b1f5" + }, + "http-results/text-4-3-standard.after.health": { + "bytes": 453, + "sha256": "4b9a34a2eec3165a5c1ef2da3e32af6676938546b56f6c54a64d8d3679bb3540" + }, + "http-results/text-4-3-standard.after.metrics": { + "bytes": 32096, + "sha256": "7a2952c94a5c59ab9a56f89376b507e9ef11cb3d0b2246dc54f61c77d6b6bcc2" + }, + "http-results/text-4-3-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "http-results/text-4-3-standard.before.health": { + "bytes": 453, + "sha256": "4b9a34a2eec3165a5c1ef2da3e32af6676938546b56f6c54a64d8d3679bb3540" + }, + "http-results/text-4-3-standard.before.metrics": { + "bytes": 909, + "sha256": "ade7ba6037d5aa112151f57dd04782d956aecb2be47f22cf4fc9c4d083f2413a" + }, + "http-results/text-4-3-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "http-results/text-4-3-standard.cold-2048.client.log": { + "bytes": 296, + "sha256": "96f7200a20c9b35a1d0e0026a5683b9bb63a5dcf47ecccd80751adc0bb98a1a5" + }, + "http-results/text-4-3-standard.cold-2048.json": { + "bytes": 26084, + "sha256": "928120efa4bf266221f356ade434c8e4410768fa2cf290b9ee5c30b0bda54cef" + }, + "http-results/text-4-3-standard.cold-8192.client.log": { + "bytes": 296, + "sha256": "fb594fb0825639c9476210ea3501486ee75eda5db848397d27402bc07fda6f86" + }, + "http-results/text-4-3-standard.cold-8192.json": { + "bytes": 62934, + "sha256": "4c01a217717702166c33ee7857c616aa3532cf0e61bc97c8528262b0efccc018" + }, + "http-results/text-4-3-standard.concurrent.client.log": { + "bytes": 3242, + "sha256": "a02ad302653eb2a160c860cd97940edd484abe41ee5fdba559c2200189252df3" + }, + "http-results/text-4-3-standard.concurrent.json": { + "bytes": 285671, + "sha256": "e4fa010598e6960e5803861d64ff670ac8420eeb929d15b5a139c66d7d22a40c" + }, + "http-results/text-4-3-standard.server.log": { + "bytes": 50891, + "sha256": "9f2095c3c2dd490c74f3ea5484289f35114770b65c51aed23fb86149904a00bf" + }, + "http-results/text-4-4-pair.after.health": { + "bytes": 303, + "sha256": "c6f4ac3c78ea6f6276ceff6e8b7985336950681a1823e616465aae6be499f5b8" + }, + "http-results/text-4-4-pair.after.metrics": { + "bytes": 12306, + "sha256": "69e9b9e0a0e26ab3a79ff750984a01d6b39ff7a96b3c6022956c7467253ade94" + }, + "http-results/text-4-4-pair.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/text-4-4-pair.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "http-results/text-4-4-pair.before.metrics": { + "bytes": 12128, + "sha256": "7a0f69eeb8775379bacfc86cb7f9f5ca661fb411d5981970b034bd74d168d089" + }, + "http-results/text-4-4-pair.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/text-4-4-pair.cold-2048.client.log": { + "bytes": 350, + "sha256": "e2fe1a73e404b85f729f90192fb705340d1d850f3e92bb2ce3dad8b8b5581fcf" + }, + "http-results/text-4-4-pair.cold-2048.json": { + "bytes": 20296, + "sha256": "976ea6aa15e490f9f25fe20056b12def96175307c3fcccf78f7cb259eaffba07" + }, + "http-results/text-4-4-pair.cold-8192.client.log": { + "bytes": 348, + "sha256": "fb432b4276da4abfb875ad84320f6b3b8a59a4b783157b5848c8b742e42128eb" + }, + "http-results/text-4-4-pair.cold-8192.json": { + "bytes": 57147, + "sha256": "2df86a35f6cbf55f16ce80d00ee3e4b55db7bcec60c025bcd9956db0bda24b11" + }, + "http-results/text-4-4-pair.concurrent.client.log": { + "bytes": 3225, + "sha256": "ca0e0ff7a9159534cd408da1462a13f2233718bd1f2a15ea2f3610f5d19ca015" + }, + "http-results/text-4-4-pair.concurrent.json": { + "bytes": 282974, + "sha256": "e400828bd0335e3ecd73f3faa048c4069fdf15c2e12f29ba1f0c831e7355b9af" + }, + "http-results/text-4-4-pair.server.log": { + "bytes": 13745, + "sha256": "2d899eb63007382b348bb13a70a245792633042bfe2ebc984bece39a752bc072" + }, + "http-results/text-4-5-current.after.health": { + "bytes": 303, + "sha256": "602868efc17450aa1967781d4377e8ea2b2a697a8aa46b5975d6927ae9578894" + }, + "http-results/text-4-5-current.after.metrics": { + "bytes": 12306, + "sha256": "6e3890f89c45fa17cb033d64f2d58c909835588e58e8396f7f87a1302e9027e1" + }, + "http-results/text-4-5-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/text-4-5-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "http-results/text-4-5-current.before.metrics": { + "bytes": 12128, + "sha256": "78d5d25064fddeea2fef1c12a8eb507f9f9d20e2b70aebe640ab8e9d10479f1a" + }, + "http-results/text-4-5-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/text-4-5-current.cold-2048.client.log": { + "bytes": 348, + "sha256": "b988118142b64e8c3c2edd33aaab1e5d9763e59514839e3ba4623b6f0db44705" + }, + "http-results/text-4-5-current.cold-2048.json": { + "bytes": 20295, + "sha256": "91dbc90983c79e7f4fc4689b4abf22190817d90c0ef524f933564a00f338842e" + }, + "http-results/text-4-5-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "631738d652676d03ae528ee698e4dc374c8ebb5a3f08b9a9ce77e9f20646a9a5" + }, + "http-results/text-4-5-current.cold-8192.json": { + "bytes": 57143, + "sha256": "e70a5726664b1c2cd377433acb2414412129625f5756046c15b9928672339d22" + }, + "http-results/text-4-5-current.concurrent.client.log": { + "bytes": 3240, + "sha256": "951ad31ea4fe439ec1a0994e7f8fd037caf2eaa16ec7699bb1192860ba8bcafe" + }, + "http-results/text-4-5-current.concurrent.json": { + "bytes": 282915, + "sha256": "f538575e9b2f1fae482a87c5a69742ffd15ed5e7ca92bcec6397453c2bfd4981" + }, + "http-results/text-4-5-current.server.log": { + "bytes": 13669, + "sha256": "9faeafbc875f5f6679351ee0ba05b8347e5c239fd668732d963a6f9d71073a5e" + }, + "http-results/vl-4-0-current.after.health": { + "bytes": 303, + "sha256": "6be7062b427e5dacf8398716128aed6806ff70c753747cebf28d879f91d5c005" + }, + "http-results/vl-4-0-current.after.metrics": { + "bytes": 12306, + "sha256": "6aa607548c1408291794efca5efe852be794d7090912ef23465e68abf0d16bb2" + }, + "http-results/vl-4-0-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/vl-4-0-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "http-results/vl-4-0-current.before.metrics": { + "bytes": 12128, + "sha256": "11865eba546442f6f572b01d99c9eda58ef43dc8e49ce242b289ee14722fb216" + }, + "http-results/vl-4-0-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/vl-4-0-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "c2e93b3c4266d32dffe3fa352860f22d66cd7fabb7cfb664535110ca9f157bae" + }, + "http-results/vl-4-0-current.cold-2048.json": { + "bytes": 20278, + "sha256": "61fe01e1064eb4e60575b24b6365e8fd60cde1ee2e83bdf268351abb5d1d4ae4" + }, + "http-results/vl-4-0-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "6556bd4670562223bf7a1ce5d55721591e585a5ec3b54d16f18a8f6b1c998f8a" + }, + "http-results/vl-4-0-current.cold-8192.json": { + "bytes": 57145, + "sha256": "1363ef6bf24272487c0ae2fbaa26b8fdb5c01ac6efe46c755772e63a93d429ea" + }, + "http-results/vl-4-0-current.concurrent.client.log": { + "bytes": 3228, + "sha256": "9286dadf9f05fa05a6b26f6938ea6c7e60f624bb5f8afd2cab4a225bf4803026" + }, + "http-results/vl-4-0-current.concurrent.json": { + "bytes": 289148, + "sha256": "a595a08c1e7d26d27cfc4e5a937c101f61117f04e18467b00887212fd1ee5bf9" + }, + "http-results/vl-4-0-current.server.log": { + "bytes": 13641, + "sha256": "ef40fe297657b1f1a730f15589c18f871139d84e15ccfe4c9a4beef381807f62" + }, + "http-results/vl-4-1-standard.after.health": { + "bytes": 457, + "sha256": "ddd816f4e9d2c97ca67ca18fc173865751e14813a790ffaf1c1183ffd0e3fac2" + }, + "http-results/vl-4-1-standard.after.metrics": { + "bytes": 32159, + "sha256": "d6dadf4b325194ea1d0df9303c5b19cd28e110f3eeb21f63167d6d95c1e55313" + }, + "http-results/vl-4-1-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "http-results/vl-4-1-standard.before.health": { + "bytes": 457, + "sha256": "ddd816f4e9d2c97ca67ca18fc173865751e14813a790ffaf1c1183ffd0e3fac2" + }, + "http-results/vl-4-1-standard.before.metrics": { + "bytes": 912, + "sha256": "dd4acb9f3449e184831844eb162b1edb74e78b7badea36792c18e334a1a1873b" + }, + "http-results/vl-4-1-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "http-results/vl-4-1-standard.cold-2048.client.log": { + "bytes": 296, + "sha256": "b33d0ae8330bc241718491f3edec228ea7e4c602ad2e45c3a5e8a89b5a6fc993" + }, + "http-results/vl-4-1-standard.cold-2048.json": { + "bytes": 26081, + "sha256": "07a5c7f784a2ea17baa2d804708d06f6c24cec7f1f4a54e55ffb581fe78e55a6" + }, + "http-results/vl-4-1-standard.cold-8192.client.log": { + "bytes": 295, + "sha256": "40f13562c53a6d05295e91dbee87c7506383135960d3dd9c21ad0e63e0ea03e6" + }, + "http-results/vl-4-1-standard.cold-8192.json": { + "bytes": 62952, + "sha256": "73b8c68247b5779c754931beea8d9f031d3298ae51f414549d37b020a66d518e" + }, + "http-results/vl-4-1-standard.concurrent.client.log": { + "bytes": 3219, + "sha256": "bd46d94410ec6af081bed04787e5c8226c1d8f5d21d0aafb14f78dd901323134" + }, + "http-results/vl-4-1-standard.concurrent.json": { + "bytes": 285423, + "sha256": "fe742faf5ec5a6d6150b81cc93d9891e2fbb89cbb5c0dbe7b590d02f49cfd9b4" + }, + "http-results/vl-4-1-standard.server.log": { + "bytes": 50969, + "sha256": "cb0f6f7f0343d4a0c036a07e2b6004fd3dca2eea8bccb8e598848dca9c4b931c" + }, + "http-results/vl-4-2-standard.after.health": { + "bytes": 457, + "sha256": "ddd816f4e9d2c97ca67ca18fc173865751e14813a790ffaf1c1183ffd0e3fac2" + }, + "http-results/vl-4-2-standard.after.metrics": { + "bytes": 32148, + "sha256": "1b9a49e67e7d452f685b4228690e6e62d6216d4afb4e3498b9106dd94d25bfc3" + }, + "http-results/vl-4-2-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "http-results/vl-4-2-standard.before.health": { + "bytes": 457, + "sha256": "ddd816f4e9d2c97ca67ca18fc173865751e14813a790ffaf1c1183ffd0e3fac2" + }, + "http-results/vl-4-2-standard.before.metrics": { + "bytes": 912, + "sha256": "2ca902329cc406ddab4a22917dbfd75ff2dd46ab98a32304cfee627e5e39c472" + }, + "http-results/vl-4-2-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "http-results/vl-4-2-standard.cold-2048.client.log": { + "bytes": 297, + "sha256": "3943f8e96d62cce1fce5866afa0db1ee804a216c86dd4f1d97e99124c86c4ad6" + }, + "http-results/vl-4-2-standard.cold-2048.json": { + "bytes": 26078, + "sha256": "6b0aadd900aecfd94e7edcf41809b1a55efcfd99c5117c6fa921aa8300369b81" + }, + "http-results/vl-4-2-standard.cold-8192.client.log": { + "bytes": 295, + "sha256": "ef0b25d5e93ef2d1edc89e8bce8535a784d0cdc865c63750b35ad6b18450ab25" + }, + "http-results/vl-4-2-standard.cold-8192.json": { + "bytes": 62947, + "sha256": "0f0f1d42041c1b36c15078449ad39eee79c52745b98a5d3dcd6e17610cbe8002" + }, + "http-results/vl-4-2-standard.concurrent.client.log": { + "bytes": 3220, + "sha256": "c692e7879d215df95d388c83672ce27dda29d78f3fd73c470d05eb7fb1a5312b" + }, + "http-results/vl-4-2-standard.concurrent.json": { + "bytes": 285646, + "sha256": "7c810f7d8a77f15d0970b7016a7732e17c9a22122206cfaea3d383f59e862764" + }, + "http-results/vl-4-2-standard.server.log": { + "bytes": 50969, + "sha256": "ed0ea0118772c446dd511feeae5650d2e5618ac353df02e26b5df0e2f46a4222" + }, + "http-results/vl-4-3-current.after.health": { + "bytes": 303, + "sha256": "b76da36250c4d5f6687ee0087d360eb5031c15529606a045e337001d0feec7d2" + }, + "http-results/vl-4-3-current.after.metrics": { + "bytes": 12305, + "sha256": "8f3a02c8d6a74725461341ce74244367f70040e57123bc1c4171dedb1ca9bd49" + }, + "http-results/vl-4-3-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/vl-4-3-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "http-results/vl-4-3-current.before.metrics": { + "bytes": 12128, + "sha256": "01d10c7d9e03abcb4d928582033c5d7107220a4193a498d0974d642835adb1ee" + }, + "http-results/vl-4-3-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "http-results/vl-4-3-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "60f435f0abd8537208b684c2a70125f7047c919d7430f5208fa8a92f32ee4931" + }, + "http-results/vl-4-3-current.cold-2048.json": { + "bytes": 20279, + "sha256": "affe00c4a3c4ef931f0413a191b19730bed2dff794aba52d3938486eabb65269" + }, + "http-results/vl-4-3-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "433dff20e6a5f8d2a1892b651e83063603dbe9a73a35cd372f3bca87a5f79672" + }, + "http-results/vl-4-3-current.cold-8192.json": { + "bytes": 57146, + "sha256": "ee50fa386246da4b2f8fc1e4ae690affcbd86f197fe3f955085c4b33b0468e29" + }, + "http-results/vl-4-3-current.concurrent.client.log": { + "bytes": 3223, + "sha256": "2c0d8c7115ad692169368fe0e1957bc37b54490c035a25b665f089f41932cc38" + }, + "http-results/vl-4-3-current.concurrent.json": { + "bytes": 289087, + "sha256": "58cc544b81b0e5b88e7011467dc3946747ac8f062dace116c5921c5ec9368bca" + }, + "http-results/vl-4-3-current.server.log": { + "bytes": 13641, + "sha256": "d853680caa98301903e6b435d100ab71ed2c2a680b58ddbcc6e5d39195332f36" + }, + "http.log": { + "bytes": 22106, + "sha256": "d3e10fb147dd34b1153da5298f1ecf7d95e931561ed278e58575fb925e7070f1" + }, + "proof/4-ffn-1.log": { + "bytes": 68, + "sha256": "0a440fb1c1136a2a05e490b1399243c0853d379a6096c94a1836e58013fd3e94" + }, + "proof/4-ffn-15.log": { + "bytes": 70, + "sha256": "13dcb26bc1cbaef6fd249e13695598178e8d82dd748b759c9195aa1f8ee0666a" + }, + "proof/4-ffn-16.log": { + "bytes": 70, + "sha256": "754e50ddb5f087244dcd950de115031d09a47f03704c67b094fe8c090a98f426" + }, + "proof/4-ffn-17.log": { + "bytes": 70, + "sha256": "516ca1b168bae8415125d0c1080ed69400c5ff0f7f4918d6c7883cf942ddad1f" + }, + "proof/4-ffn-2.log": { + "bytes": 68, + "sha256": "21cb81ddf8af88aee84dd9d8794a8f69bd4b8536e592c916fcfb1cc536dfbffc" + }, + "proof/4-ffn-3.log": { + "bytes": 68, + "sha256": "29a2936aade20e2365cdc224525b4347ecdb32ad0d575faef58f84f50fca22d5" + }, + "proof/4-ffn-31.log": { + "bytes": 70, + "sha256": "5d625695b2974141ea2f3dd10af276908b6c02a6b676a3c597193153e1baad59" + }, + "proof/4-ffn-32.log": { + "bytes": 70, + "sha256": "408d51c9202962b5c06f0e9050248cc77a665d440d70c4a0339f6218c3253034" + }, + "proof/4-ffn-33.log": { + "bytes": 71, + "sha256": "7b49974e82e5df448e60a14c5eaeb218f98c0a51575e70adcf5293b9e40c132d" + }, + "proof/4-ffn-4.log": { + "bytes": 69, + "sha256": "99c6bbb75a119148c3c8f0aaf36834f68e911e40ecc95d4273adb7a510a95c82" + }, + "proof/4-ffn-5.log": { + "bytes": 69, + "sha256": "5956e81029fa558ab5de1cb0fc61de5ef9fdbc6aed8d2f3b1410363e10afd793" + }, + "proof/4-ffn-63.log": { + "bytes": 71, + "sha256": "2985d58dcfc5f0c4ba03d720319b615abc5e27e538d6d9626fb7fce3ad92d9b3" + }, + "proof/4-ffn-64.log": { + "bytes": 71, + "sha256": "37f53a635bd8378fd0adeb6628e41003ccbf9fd887b92fc717bd8bb966c661bc" + }, + "proof/4-ffn-7.log": { + "bytes": 69, + "sha256": "53286a376acebe7869a2ec3442138018914c1339b6360335aaad0c0c531bc396" + }, + "proof/4-ffn-8.log": { + "bytes": 69, + "sha256": "6fc70da8e674c660855f6a4d8c0141eb25e99c38605b9c86ad43abd0b48973ec" + }, + "proof/4-ffn-nf7f80.log": { + "bytes": 71, + "sha256": "7b49974e82e5df448e60a14c5eaeb218f98c0a51575e70adcf5293b9e40c132d" + }, + "proof/4-ffn-nf7fc1.log": { + "bytes": 71, + "sha256": "7b49974e82e5df448e60a14c5eaeb218f98c0a51575e70adcf5293b9e40c132d" + }, + "proof/4-ffn-nfff80.log": { + "bytes": 71, + "sha256": "7b49974e82e5df448e60a14c5eaeb218f98c0a51575e70adcf5293b9e40c132d" + }, + "proof/4-kv-1.log": { + "bytes": 66, + "sha256": "fe43a02ddd7448f7b611e9873ab20f82c9689ba2d69e4d04df0cbf3497101ca5" + }, + "proof/4-kv-15.log": { + "bytes": 68, + "sha256": "b45fbcfcbbd9a66ff0339f08e941f7d1d95b723e197ff74590fdd06d29acd234" + }, + "proof/4-kv-16.log": { + "bytes": 68, + "sha256": "e88de31c2703c133832558c4b4bac7025a89ca8de9736c2703e3e6d95652859e" + }, + "proof/4-kv-17.log": { + "bytes": 68, + "sha256": "e17e850d46238db91997e62ecefe8fdd5be9ee63045d066cb36db620b2226453" + }, + "proof/4-kv-2.log": { + "bytes": 66, + "sha256": "72deb27fffed515c3b278354c14e7a6a5d386fb7858468051585a27333036b0c" + }, + "proof/4-kv-3.log": { + "bytes": 66, + "sha256": "906c0a2325ec3967e3c934db49e327b72ed4a8687bea6be6f7d2d3b9a0708a7a" + }, + "proof/4-kv-31.log": { + "bytes": 68, + "sha256": "d042703ea46aa5eb7eb1bed62d474b63c2a326e14cc8cbdd43cd56fdfa1c295c" + }, + "proof/4-kv-32.log": { + "bytes": 68, + "sha256": "5b766a30832287c2b52617eadc4bc4c1bd94be2cfa89b60b333b5a66da1c48e3" + }, + "proof/4-kv-33.log": { + "bytes": 68, + "sha256": "3c0ebba9793619da4e4b5749367b9af81957c42f280bdba35e99aa3c78fa97f8" + }, + "proof/4-kv-4.log": { + "bytes": 66, + "sha256": "cfd917124f13fd7b8edd0796d8046847f207d7eba3221ae13ffa4109fb4d9695" + }, + "proof/4-kv-5.log": { + "bytes": 67, + "sha256": "4368f4840d1c8aee47a4be5c6f4f10b1bf7f8cf04916e5ca81d4f4590d573de7" + }, + "proof/4-kv-63.log": { + "bytes": 69, + "sha256": "d43613b96c8ef0a476966460e3f087c63bae5206859d0dd339b4ce565b607268" + }, + "proof/4-kv-64.log": { + "bytes": 69, + "sha256": "bd1dcf2577e05ef0d1d2f0a6151b9155d3cfeae3b9217c958c4399794899dd50" + }, + "proof/4-kv-7.log": { + "bytes": 67, + "sha256": "7fd31c93457566ea68aa10deb58716b7dd22f21ebc52ede7b59efda1946cf9ea" + }, + "proof/4-kv-8.log": { + "bytes": 67, + "sha256": "164bf999b45eba6814a03eacd622ed46f23a107806b255e1d870e7c11b3047b4" + }, + "proof/4-kv-nf7f80.log": { + "bytes": 68, + "sha256": "3c0ebba9793619da4e4b5749367b9af81957c42f280bdba35e99aa3c78fa97f8" + }, + "proof/4-kv-nf7fc1.log": { + "bytes": 68, + "sha256": "3c0ebba9793619da4e4b5749367b9af81957c42f280bdba35e99aa3c78fa97f8" + }, + "proof/4-kv-nfff80.log": { + "bytes": 68, + "sha256": "3c0ebba9793619da4e4b5749367b9af81957c42f280bdba35e99aa3c78fa97f8" + }, + "proof/6-ffn-1.log": { + "bytes": 68, + "sha256": "4cdd5b755a373447f583a80f0cf82efbebfc8eaebf8c01985a48a219c38e2cf0" + }, + "proof/6-ffn-15.log": { + "bytes": 70, + "sha256": "7ac345eef2bd92c1c93d1f159c5bea9677bdd357a4a497a08c2277d4e5092349" + }, + "proof/6-ffn-16.log": { + "bytes": 70, + "sha256": "f9ba1a57e3396b5f17bc6b15c1a87a1cb0dd66f0f0e2c79d6bedf6d31b3d9a11" + }, + "proof/6-ffn-17.log": { + "bytes": 70, + "sha256": "07540b5d73f9d424fd6a31aca8f0e4df33fe85d124cf38cf634a4da323d4910e" + }, + "proof/6-ffn-2.log": { + "bytes": 68, + "sha256": "99f6e747a32ef7e4fc956bd602066800364e7a9afee4aaebe48d7cb5e0c1a29b" + }, + "proof/6-ffn-3.log": { + "bytes": 68, + "sha256": "f103fdaba792757c755c1d5b834f6be2b832013074af24079bc46bc9a27eed5f" + }, + "proof/6-ffn-31.log": { + "bytes": 70, + "sha256": "5bfde70efd999647b522d2e6fa96af6592e16525cf6f2f3a0fac19d43476be6d" + }, + "proof/6-ffn-32.log": { + "bytes": 70, + "sha256": "9b8898fdc7512b73c53eda15207264da2d2099b84298ac416d7c069bfec13d79" + }, + "proof/6-ffn-33.log": { + "bytes": 71, + "sha256": "535e05bcff4898948786f8600594bb9606068485ac09bbf9176a06816ffde426" + }, + "proof/6-ffn-4.log": { + "bytes": 69, + "sha256": "5dcd1685047003c6381132cb16c9ad58d1b53a8694d8c918fa2982eb5dc435ee" + }, + "proof/6-ffn-5.log": { + "bytes": 69, + "sha256": "69797df071d57c3b4c7ed419091fe6cbb10c18cb9e5acf0e41208ec1c9fa3d75" + }, + "proof/6-ffn-63.log": { + "bytes": 71, + "sha256": "696ce1f02cc4700ec60ee662df1fe5a3a97a1886ce79e05583226a9bb0e4db3c" + }, + "proof/6-ffn-64.log": { + "bytes": 71, + "sha256": "ca26d37339d7ed3547bca3016793c2f4fae5b3146342cd416c6ff108f4cc7682" + }, + "proof/6-ffn-7.log": { + "bytes": 69, + "sha256": "bb3034e2736a8890a9534d5a1d141e11436f714a7f8f8951b34cd348f0af91ab" + }, + "proof/6-ffn-8.log": { + "bytes": 69, + "sha256": "63c849c01b54ceba39ef753989262cb895e104ff78e46fa6e2ed6657f2d32cd5" + }, + "proof/6-ffn-nf7f80.log": { + "bytes": 71, + "sha256": "535e05bcff4898948786f8600594bb9606068485ac09bbf9176a06816ffde426" + }, + "proof/6-ffn-nf7fc1.log": { + "bytes": 71, + "sha256": "535e05bcff4898948786f8600594bb9606068485ac09bbf9176a06816ffde426" + }, + "proof/6-ffn-nfff80.log": { + "bytes": 71, + "sha256": "535e05bcff4898948786f8600594bb9606068485ac09bbf9176a06816ffde426" + }, + "proof/6-kv-1.log": { + "bytes": 66, + "sha256": "c7ed815d9852caa06b4964504a6d2cb7d0560e6b35c69e4ad45fccad50e638b1" + }, + "proof/6-kv-15.log": { + "bytes": 68, + "sha256": "2a1aac83429e67211279125e2042b4eb09df95b1db78718944b866eb08f5b090" + }, + "proof/6-kv-16.log": { + "bytes": 68, + "sha256": "ff05709a1e79f562fe39d06c8fdd100407295d990add0297d550bdcf2a92a2e0" + }, + "proof/6-kv-17.log": { + "bytes": 68, + "sha256": "a33be246560376d27c2e9b3963b51d166538070294e3d5d0117a86ca3b15f73d" + }, + "proof/6-kv-2.log": { + "bytes": 66, + "sha256": "038e9bb056ea42ee2562f5a27b38bd5857f6cef6ac21e9827625cc8e20611bfd" + }, + "proof/6-kv-3.log": { + "bytes": 66, + "sha256": "6539d824cb0c82f85edc3dde2073fdf269d790ecde4e6a3762f2134ceb416aed" + }, + "proof/6-kv-31.log": { + "bytes": 68, + "sha256": "56571924afe7e83dd2f07ae374777b601f4b0c8f631a4513a86a4c9c060b0b18" + }, + "proof/6-kv-32.log": { + "bytes": 68, + "sha256": "e55cc4efe0fca1197abb18097dab7128f22fc955fde63c892d3ac958c7e0d023" + }, + "proof/6-kv-33.log": { + "bytes": 68, + "sha256": "8b22270452882348b7ca6b1e161369a107118339e8ef09f1906eb8f4bbef8cd2" + }, + "proof/6-kv-4.log": { + "bytes": 66, + "sha256": "35ab4f1a3c67dd907d24efc56b7c1db42ab002d48324f57ca8218c89ff5f0b24" + }, + "proof/6-kv-5.log": { + "bytes": 67, + "sha256": "40e89ffee251fac917ac68dbafe2135a16008f584e83c2f36d2848492c5e49a3" + }, + "proof/6-kv-63.log": { + "bytes": 69, + "sha256": "c599c82ed56ab28559f8bd63bf0828f9e60128411732dafe5d6b191724672304" + }, + "proof/6-kv-64.log": { + "bytes": 69, + "sha256": "b163baa7ac2f82faecec8108f59eebd5a82313d3be554f150188d43ab94b4c43" + }, + "proof/6-kv-7.log": { + "bytes": 67, + "sha256": "fbd0176b1cb83cb6219d2e3c166a937abec874862737767094995165ea8cf6e9" + }, + "proof/6-kv-8.log": { + "bytes": 67, + "sha256": "a28e3cdc0510b03de588a644ca6e3dee19946a0b951bdb64deeccd664730d7fa" + }, + "proof/6-kv-nf7f80.log": { + "bytes": 68, + "sha256": "8b22270452882348b7ca6b1e161369a107118339e8ef09f1906eb8f4bbef8cd2" + }, + "proof/6-kv-nf7fc1.log": { + "bytes": 68, + "sha256": "8b22270452882348b7ca6b1e161369a107118339e8ef09f1906eb8f4bbef8cd2" + }, + "proof/6-kv-nfff80.log": { + "bytes": 68, + "sha256": "8b22270452882348b7ca6b1e161369a107118339e8ef09f1906eb8f4bbef8cd2" + }, + "proof/8-ffn-1.log": { + "bytes": 68, + "sha256": "de95bbd2f027c23b1c54e57943a49f81b07c3a61bafe9dda8d4640a73f8e3aac" + }, + "proof/8-ffn-15.log": { + "bytes": 70, + "sha256": "d839986f7d370640775e8d931cbcab71be7650ba4c3acb3d89dd721d1ad33224" + }, + "proof/8-ffn-16.log": { + "bytes": 70, + "sha256": "0de65ffc42a6eb589bcb6b6ea0235f236dd3eb82a2da87d71d53c12ed5c2f87e" + }, + "proof/8-ffn-17.log": { + "bytes": 70, + "sha256": "b48882c7f0eeca32623877bac20c2f984a62bdfa3e655aa2a7510563cf41b8f7" + }, + "proof/8-ffn-2.log": { + "bytes": 68, + "sha256": "dd52b55f8e06a8873ee915bd5394ae0341d76bf1a7a6f238bc8361aed6be9fda" + }, + "proof/8-ffn-3.log": { + "bytes": 68, + "sha256": "601c40a27ae2ad182df0c0e2f8d16011d91bbfbdf069fd959d7603f948fe576a" + }, + "proof/8-ffn-31.log": { + "bytes": 70, + "sha256": "cc13d09d3e3fdc3fdca68e72abc2cefa5b7b67fa80f398b156e609bc55c0a0ed" + }, + "proof/8-ffn-32.log": { + "bytes": 70, + "sha256": "d35a667f5ccbf6f3cd3cbec99439f45ca885fd7a017dd6a121b2c4b7f47bdb52" + }, + "proof/8-ffn-33.log": { + "bytes": 71, + "sha256": "7dd6fee5464bd09733b39eafb4ca9fc2adcf5fb3a4248b8dfc4cc53a010a3527" + }, + "proof/8-ffn-4.log": { + "bytes": 69, + "sha256": "db029a92c9fb380d714a91d3a765d61d59f33ee2b636533a2b62bb6c138ca4a4" + }, + "proof/8-ffn-5.log": { + "bytes": 69, + "sha256": "793fe7ec32771ae0a800c48fa602c121d479fcc9e16975ae25a03bdc796424d9" + }, + "proof/8-ffn-63.log": { + "bytes": 71, + "sha256": "8b203d4126444e54af1c0b89d31a88784ad0f1656f06f11e0d11de3a1a54b6a0" + }, + "proof/8-ffn-64.log": { + "bytes": 71, + "sha256": "fdc04e0afa37428dfbee97275570a7ed66f59ed45c9a86a7aebe7fc098698f0d" + }, + "proof/8-ffn-7.log": { + "bytes": 69, + "sha256": "3d856d211329c919e21cc957632ea76c25b74c9c71df8bd5c7cff1736ba1284f" + }, + "proof/8-ffn-8.log": { + "bytes": 69, + "sha256": "9cc900edd38de2296509fb9c11d0265e4e47e3c5d669dd92917cb1ce7a74d5d4" + }, + "proof/8-ffn-nf7f80.log": { + "bytes": 71, + "sha256": "7dd6fee5464bd09733b39eafb4ca9fc2adcf5fb3a4248b8dfc4cc53a010a3527" + }, + "proof/8-ffn-nf7fc1.log": { + "bytes": 71, + "sha256": "7dd6fee5464bd09733b39eafb4ca9fc2adcf5fb3a4248b8dfc4cc53a010a3527" + }, + "proof/8-ffn-nfff80.log": { + "bytes": 71, + "sha256": "7dd6fee5464bd09733b39eafb4ca9fc2adcf5fb3a4248b8dfc4cc53a010a3527" + }, + "proof/8-kv-1.log": { + "bytes": 66, + "sha256": "c732a5c57edd7ee8e68c868680f77b6add1833b1c164525feb2a786dfafcdfce" + }, + "proof/8-kv-15.log": { + "bytes": 68, + "sha256": "6c3ec60344fb8df4ae18859fa52782d55958aa959cbb776feabbdebc3b9f62ad" + }, + "proof/8-kv-16.log": { + "bytes": 68, + "sha256": "244e6b7e5b5897a9f94609d77bcf25ffa0533050ff9e185bae0d6d7d67850aba" + }, + "proof/8-kv-17.log": { + "bytes": 68, + "sha256": "148ff59ea5186337575b8c87028c9537bc0eeaa0f15b2472875d5f0eff3270e8" + }, + "proof/8-kv-2.log": { + "bytes": 66, + "sha256": "4a8c39bed3908422ad3d004cc8a51e76c4a867e55d7ceae6eb7889286ff88db6" + }, + "proof/8-kv-3.log": { + "bytes": 66, + "sha256": "62b319f41629686cd3b829270b946d78c2822580fbbc9143ddb0ecacda17b537" + }, + "proof/8-kv-31.log": { + "bytes": 68, + "sha256": "7c32a91554b6eb73dd6af27e2e44b2f9e6a381dfc076f921c41b6cb29982b51e" + }, + "proof/8-kv-32.log": { + "bytes": 68, + "sha256": "fd1da0acd80415a00a8e9f0e275fbc11a66952fdabe8a8b338b7a5177cb32ef1" + }, + "proof/8-kv-33.log": { + "bytes": 68, + "sha256": "dc2fea731fbe954b45e80e0b84559fa1b199f965fa0fe9ed56c590fa57d05086" + }, + "proof/8-kv-4.log": { + "bytes": 66, + "sha256": "ab57f621b838f885b8304a1182feb09cad88f8626e13abc97c9c2ea22bd8c560" + }, + "proof/8-kv-5.log": { + "bytes": 67, + "sha256": "143416f4b1c097a1ae93cc63eab31ccd79160c0e29ce6e938d2baf632a7bb399" + }, + "proof/8-kv-63.log": { + "bytes": 69, + "sha256": "c84325439a391804c025f3827b1531c9837a8859c5fc6ac8070f2eb638d8791e" + }, + "proof/8-kv-64.log": { + "bytes": 69, + "sha256": "d3a38b959675b51e641f19ab657ed2893e0e92948a0f3f4551eb3d728d86547b" + }, + "proof/8-kv-7.log": { + "bytes": 67, + "sha256": "aa2ac2da5ffce884af13e1ac72000ceacd0867e5bf2c16ac61feb568519e881a" + }, + "proof/8-kv-8.log": { + "bytes": 67, + "sha256": "add20ea8714e8fc7edb953d1dca4a7b13582da911aee541d651bca3932d914e7" + }, + "proof/8-kv-nf7f80.log": { + "bytes": 68, + "sha256": "dc2fea731fbe954b45e80e0b84559fa1b199f965fa0fe9ed56c590fa57d05086" + }, + "proof/8-kv-nf7fc1.log": { + "bytes": 68, + "sha256": "dc2fea731fbe954b45e80e0b84559fa1b199f965fa0fe9ed56c590fa57d05086" + }, + "proof/8-kv-nfff80.log": { + "bytes": 68, + "sha256": "dc2fea731fbe954b45e80e0b84559fa1b199f965fa0fe9ed56c590fa57d05086" + }, + "proof/current-normal70.log": { + "bytes": 99027, + "sha256": "d3631ea9b1f37e25ef13a43aabfe9a1ff54875170a04fb2acbdb637cafbbbef6" + }, + "proof/current-shared-tiles.log": { + "bytes": 3139, + "sha256": "2767db9832e7801652aafdb998dde8ff69451fade9aff318b39d18e28d1ad1bd" + }, + "proof/events.json": { + "bytes": 81478, + "sha256": "3604552393d3dc3713d9a29dfb9b6312c3d2116def25b48cb054c96a714133d5" + }, + "proof/manifest.json": { + "bytes": 1557, + "sha256": "dd6f090b528cb6163168810fa321b1177f3ecbbc1c3373f6595996974ccb746e" + }, + "proof.log": { + "bytes": 68771, + "sha256": "82eb11b8d7de79cccf57da5616a1a86716777d1fa60e1465c82bcbac826c8a2c" + }, + "receipt.json": { + "bytes": 2915, + "sha256": "c5b5d34650ece84663955d12263e4274308cccc199bd66f32ec052c0ff3c467d" + } + }, + "raw_retention": "Raw logs, per-request token timing arrays, repeated metrics/comparisons, health/settings/metrics and proof outputs remain in the frozen local result directory. This receipt retains per-leg/per-repetition TPS, end-to-end TPS, makespan, per-stream TPS/TTFT, failure counts, strict token/cache evidence and memory; evidence is deduplicated by reference. Hashes retain the omitted raw provenance." +} diff --git a/docs/receipts/lfm3/m5-current68-standard.json b/docs/receipts/lfm3/m5-current68-standard.json new file mode 100644 index 0000000000..bc51488e7d --- /dev/null +++ b/docs/receipts/lfm3/m5-current68-standard.json @@ -0,0 +1,11759 @@ +{ + "schema_version": 1, + "scope": "Current public runtime compared with pinned standard on Apple M5 Ultra: text1.7B and VL2B6/8bit,2k/8k text-only cold prompts,256-token solo/four-stream code/prose. No performance gate pass.", + "status": { + "actual_exit": 0, + "server_legs": 16, + "primary_measured_requests": 480, + "primary_repetitions": 192, + "cold_requests": 32, + "all_primary_256_tokens": true, + "all_cold_16_tokens_cached0": true, + "native_concurrent_to_solo_equal": 240, + "native_serial_equal": 32, + "owned_server_cleanup_waits_completed": 16, + "native_graceful_exit0": 8, + "standard_owned_SIGTERM_exit_minus15": 8, + "performance_accepted": false, + "conclusion": "Current public throughput and cold prefill remain slower than the measured applicable standard in all four model/width cells." + }, + "host": { + "gpu": "Apple M5 Ultra", + "gpu_cores": 64, + "physical_memory_bytes": 274877906944, + "os": "ProductName:\t\tmacOS\nProductVersion:\t\t27.0\nBuildVersion:\t\t26A428\n" + }, + "provenance": { + "public_head_at_preparation": "07a7cb4cbc137259c1f53373e59e6d88d8ff8074", + "native_provenance": "Current3d4 applied artifact built at runtime fixdd7263c26, source hashes recorded attention-range.json; git diff dd7263c26..07a7cb4 -- zig empty; later docs commits do not substitute a rebuild.", + "native_artifact": { + "current": { + "path": "$RUN/bin/tensorfold-native-current", + "sha256": "3d4a3daeda697420bfbba6d81972896c2bf9e31efaf6f12321ee8487a299354b" + } + }, + "versions": { + "python": "3.13.15 (main, Sep 1 2026, 14:05:45) [Clang 22.1.3 ]", + "packages": { + "shellingham": "1.5.4", + "uvicorn": "0.54.0", + "xxhash": "4.0.2", + "filelock": "4.2.4", + "mlx": "0.32.4", + "mlx-lm": "0.32.2", + "starlette": "1.10.0", + "huggingface_hub": "2.3.0", + "click": "8.5.0", + "packaging": "26.3", + "protobuf": "7.36.2", + "tokenizers": "0.23.3", + "Pygments": "2.21.0", + "pyarrow": "26.0.0", + "sounddevice": "0.5.6", + "requests": "2.34.2", + "rich": "15.0.0", + "certifi": "2026.10.23", + "yarl": "1.25.1", + "websockets": "17.3", + "mlx-vlm": "0.7.11", + "opentelemetry-api": "1.45.1", + "pandas": "3.1.0", + "datasets": "5.1.0", + "sentencepiece": "0.2.2", + "propcache": "0.5.4", + "llguidance": "1.9.1", + "numpy": "2.5.4", + "aiohappyeyeballs": "2.7.1", + "cryptography": "50.0.2", + "pycparser": "3.12", + "tzdata": "2026.5", + "pillow": "13.0.0", + "fsspec": "2026.7.0", + "hf-xet": "1.7.0", + "safetensors": "0.8.0", + "miniaudio": "1.71", + "tqdm": "4.70.1", + "urllib3": "2.8.0", + "h11": "0.16.0", + "pydantic_core": "2.50.1", + "mdurl": "0.1.2", + "python-multipart": "0.0.32", + "mlx-metal": "0.32.4", + "PyYAML": "6.0.3", + "httpcore2": "2.16.0", + "opencv-python": "5.0.0.93", + "truststore": "0.10.5", + "frozenlist": "1.8.0", + "cffi": "2.1.1", + "httpx2": "2.16.0", + "aiosignal": "1.4.0", + "typer": "0.27.3", + "mlx-audio": "0.6.2", + "Jinja2": "3.1.6", + "annotated-types": "0.8.0", + "regex": "2026.9.29", + "markdown-it-py": "4.2.0", + "annotated-doc": "0.0.5", + "typing-inspection": "0.4.4", + "fastapi": "0.162.0", + "aiohttp": "3.14.5", + "attrs": "26.1.0", + "multidict": "7.2.1", + "typing_extensions": "4.16.0", + "MarkupSafe": "3.0.4", + "pydantic": "2.14.1", + "python-dateutil": "3.0.0", + "multiprocess": "0.70.19", + "charset-normalizer": "3.5.2", + "anyio": "4.15.1", + "six": "1.17.0", + "scipy": "1.18.1", + "dill": "0.4.1", + "transformers": "5.21.0", + "idna": "3.20" + } + }, + "standard": { + "python": "$QUALIFICATION/standard-venv/bin/python", + "module": "mlx_vlm.server", + "source_commit": "230fc308efb577d87446306bfb8a3e9f1a129558", + "mlx": "0.32.4", + "mlx_metal": "0.32.4", + "mlx_vlm": "0.7.11", + "dependency_provenance": "Existing remote environment, installedmetadata captured CPUonly; all distributions frozen in requirements.lock. CoreMLX/VLMpins matchsource, dependency differences fromlocalM4retained.", + "memory_policy": "Unmodified MLX defaults; no memory/wired limit overrides.", + "request_context_limit": 70000, + "context_policy": "Pinned --max-kv-size70000 sets request budget; LFM3 family cache construction remains lazy and is not claimed allocation-identical to native.", + "checkpoint_context_caps": { + "text": 262144, + "vl": 128000 + } + }, + "checkpoint_files_and_revisions": { + "LFM3-1.7B-MLX-6bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-6bit", + "revision": "26be898c36b545d625d84ac0761c6c3eb664aef3", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 2423, + "sha256": "2b5879bf6d1d99b43e17726c7ee3fb4868e4c2e29c4a49784803df90f494fc2e", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 328, + "sha256": "0095ba7158e938b1e2d3082d412f00d8be85e42cbbd7608b63866f7a081cf270", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1405187179, + "sha256": "503d68675793fba64907842e176d066c8c3adca22e14c8b3538f9ffb76bd5e0e", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 50701, + "sha256": "940f7e7b4133162a48fbca232e663c32240364aea9d60f1715b62bed41a81e2c", + "verified_against_pinned_revision": true + }, + "special_tokens_map.json": { + "bytes": 92, + "sha256": "7946f864cbfb62a1b041bda937244210a1a87f4a6cab3a48b791131b2cd57f88", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908634, + "sha256": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1859, + "sha256": "b77214b9f12be74b9e5e1dab35d7dace077a31be969314bf38866fd6c8163853", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-1.7B-MLX-8bit": { + "repository": "LiquidAI/LFM3-1.7B-MLX-8bit", + "revision": "ee7667f3cfccb2a41725f91820531823fcabc6b7", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 2423, + "sha256": "4ef95e289e7105a09bf010b3ceaa015d3f01163104c039a4e7948739d5b9035b", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 328, + "sha256": "0095ba7158e938b1e2d3082d412f00d8be85e42cbbd7608b63866f7a081cf270", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1837290805, + "sha256": "2b06cea98d75527c248a0b04383f7cf0a1c72fb4ed7b45096618fc4e59d015ad", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 50701, + "sha256": "0f4f85710fd22ffb8a1977520a734cb7dc7e1f7858a0f91d9c8b56e6d36b78cc", + "verified_against_pinned_revision": true + }, + "special_tokens_map.json": { + "bytes": 92, + "sha256": "7946f864cbfb62a1b041bda937244210a1a87f4a6cab3a48b791131b2cd57f88", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908634, + "sha256": "41927bd314d17d7d7fadb44e10cd0919f7b71528d7741596d8cbd15c6945ec44", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1859, + "sha256": "b77214b9f12be74b9e5e1dab35d7dace077a31be969314bf38866fd6c8163853", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-VL-2B-MLX-6bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-6bit", + "revision": "6d71bfae121abedc75f6d531fc7f6ceb4f5e3501", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 4138, + "sha256": "4c03aa298372cbcb478eb08be0a4a59be917b4949756066ce4eac06033d200bc", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 323, + "sha256": "30b1115bcdc4e5671567a5b1255a2f3d2f5a912a79292051c6f1e9db81b5e8c4", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 1783964482, + "sha256": "838c39147541b12e86a4f1b43d6a7814d7fb4efe9d5ff8a151c85ac06f321eef", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 61255, + "sha256": "abc29f6e9809c65c5ce62e6b1754b3f103d82266adf74d6d661dd3f8cd443cb5", + "verified_against_pinned_revision": true + }, + "processor_config.json": { + "bytes": 1254, + "sha256": "f183f5a5e5b3a32e666fce05033e6cb7ca496d26e709a2cee611a13eb7d6b610", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908258, + "sha256": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1759, + "sha256": "d63d1ce2f332047e398764b4170352b96629a5194643105bfd6d70c90b750396", + "verified_against_pinned_revision": true + } + } + }, + "LFM3-VL-2B-MLX-8bit": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "revision": "83336d5be3874a2bdd3297bde77b2114dbd3a816", + "files": { + "chat_template.jinja": { + "bytes": 12129, + "sha256": "f45005c50cd2a97329722bf3a2dee9f16c04cdf825a589b280fb7e99e6827818", + "verified_against_pinned_revision": true + }, + "config.json": { + "bytes": 4138, + "sha256": "9f7dc06d04d0610c7e27fb6a110c42a3a09b686a6ec53b0eb383f30a8be5a457", + "verified_against_pinned_revision": true + }, + "generation_config.json": { + "bytes": 323, + "sha256": "30b1115bcdc4e5671567a5b1255a2f3d2f5a912a79292051c6f1e9db81b5e8c4", + "verified_against_pinned_revision": true + }, + "model.safetensors": { + "bytes": 2216068156, + "sha256": "50c9476a7f00f5349e81b42ab88b47635b9eba3c7ab5851c598d270cda4c6857", + "verified_against_pinned_revision": true + }, + "model.safetensors.index.json": { + "bytes": 61255, + "sha256": "31fe8faaf68bfc2fbce07e9d97ff47c1abad234d783ed615f2d938392b7cf6f0", + "verified_against_pinned_revision": true + }, + "processor_config.json": { + "bytes": 1254, + "sha256": "f183f5a5e5b3a32e666fce05033e6cb7ca496d26e709a2cee611a13eb7d6b610", + "verified_against_pinned_revision": true + }, + "tokenizer.json": { + "bytes": 17908258, + "sha256": "a39eed58008cbcbc09e5b629b81338a654269c64cd13348aaa6090f4db75c7e7", + "verified_against_pinned_revision": true + }, + "tokenizer_config.json": { + "bytes": 1759, + "sha256": "d63d1ce2f332047e398764b4170352b96629a5194643105bfd6d70c90b750396", + "verified_against_pinned_revision": true + } + } + } + }, + "requirements_sha256": "275b369ea72d5757b1e3f91e68ed5b746d83b5f7667adc8f5d9d28bb8933ddd6", + "packet_artifacts_sha256": { + "README.md": "7ae6f5c44bc39833b7e99921999992890724eb9e268900629bdc0c495078a128", + "cold_client.py": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2", + "config.json": "925df4aa906ae82e2e791d0608005a3808e8fab259844642c725150dbdcbe86d", + "current-source-receipt.json": "b04753edfcec89c76161e2c3e7e9ea873700179dee7d33c937e2cc53e2f0f654", + "current68-source-manifest.json": "edc67b0dd974bce6e7e21bbf315948db434c43aba530b46e4829aa1841a821ee", + "fixtures/text-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "fixtures/text-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "fixtures/text-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "fixtures/text-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd", + "fixtures/vl-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "fixtures/vl-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "fixtures/vl-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "fixtures/vl-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd", + "guard.py": "3c4118b62c3797e1871032fcbbb1bf7d390f143c2e93ddf564b309e7f49a4664", + "independent-review.json": "931231a1bc03c322be339ae7f017c692609f55209bc5e6345261ca66e8727ae1", + "model-revisions.json": "fbe3ba2fea04e926e5c237044f2c5315b31a3d991e5c9c528521dc597a924f88", + "prepare-paths.sh": "417b12ba889dfd003e4e9a6a9e77e03f9aa5e559810351b303f7bc4c7927e3e2", + "requirements.lock": "275b369ea72d5757b1e3f91e68ed5b746d83b5f7667adc8f5d9d28bb8933ddd6", + "run_existing_clients.py": "8209512004d5c8c19e77a4a1ce1032b54e492e8625faf448cfa34a07e268a2c0", + "standard-installed-metadata.json": "fb244f662eddced712fac29a25e390dad3d28f0cec79a5284f1c63cfb875b4e8", + "tools/bench_auth.py": "1896120b70d0df05646870a252697e20ce84c63e09dcba88b2a4e5a55ccb3c5b", + "tools/bench_cache.py": "24364dd13c033450f45710fc8685e2c64d6a72d6dd6e223d0d7b19aabbdc1d98", + "tools/bench_concurrent.py": "0e72e26769f082459329fc068c25f6f72a9155e422962ea54cb5d1226227752d", + "tools/bench_evidence.py": "54c28474f0804dbd120e3fb5859be0d1667ef21ca502880d0bed098ec9bb42ff", + "tools/bench_metrics.py": "26f4c3ecbb03df4f12e24d18840fea989d9ef8c63117ca11eb944ef670458a27", + "tools/bench_openai.py": "d5af4fbb756cdfb573546c0e0017652a4bb8f509d1251af683be7a3cbdd426f6", + "tools/bench_timing.py": "158f636ef0b3bed460ce19ef4872b6d8607ff7a8590562d06bae8bd55e6f44ed", + "tools/lfm3/bench_decode.py": "c011106f4f182db19371e3f91f70bcc374160786dd105f2d713a7fea07ab8c4c" + }, + "client_sha256": { + "$PACKET/tools/bench_auth.py": "1896120b70d0df05646870a252697e20ce84c63e09dcba88b2a4e5a55ccb3c5b", + "$PACKET/tools/bench_cache.py": "24364dd13c033450f45710fc8685e2c64d6a72d6dd6e223d0d7b19aabbdc1d98", + "$PACKET/tools/bench_concurrent.py": "0e72e26769f082459329fc068c25f6f72a9155e422962ea54cb5d1226227752d", + "$PACKET/tools/bench_evidence.py": "54c28474f0804dbd120e3fb5859be0d1667ef21ca502880d0bed098ec9bb42ff", + "$PACKET/tools/bench_metrics.py": "26f4c3ecbb03df4f12e24d18840fea989d9ef8c63117ca11eb944ef670458a27", + "$PACKET/tools/bench_openai.py": "d5af4fbb756cdfb573546c0e0017652a4bb8f509d1251af683be7a3cbdd426f6", + "$PACKET/tools/bench_timing.py": "158f636ef0b3bed460ce19ef4872b6d8607ff7a8590562d06bae8bd55e6f44ed", + "$PACKET/tools/lfm3/bench_decode.py": "c011106f4f182db19371e3f91f70bcc374160786dd105f2d713a7fea07ab8c4c", + "$PACKET/cold_client.py": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2" + }, + "fixture_sha256": { + "text-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "text-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "text-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "text-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd", + "vl-2048.body.json": "e6932e72d2a5ea89eec6f20a0aeb42b8e632262ecfb0482a6ac2017f17352896", + "vl-2048.ids.json": "f5edac3447405086d7ec361d6c899410ea7cbbb2ef697c27fd7486ef5ff877fe", + "vl-8192.body.json": "655407f5dc3614476ec857a0309ed5437000c2be91d3859659b256d83e1bb407", + "vl-8192.ids.json": "a9b3cdf070b9ca5ea523565bcb82c80b6af5aae71f640a8fba0460c9069d22cd" + } + }, + "protocol": { + "phase": { + "families": [ + "text", + "vl" + ], + "bits": [ + 6, + 8 + ], + "cold_tokens": [ + 2048, + 8192 + ], + "decode_tokens": 256, + "cold_output_tokens": 16, + "reps": 3, + "levels": [ + 1, + 4 + ], + "order": "family_orders", + "family_orders": { + "text": [ + "current", + "standard", + "standard", + "current" + ], + "vl": [ + "current", + "standard", + "standard", + "current" + ] + } + }, + "settings": { + "native_context": 70000, + "parallel": 4, + "prompt_cache_gib": 0, + "temperature": 0, + "thinking": false, + "prefix_cache": "disabled on both", + "port": 18085 + }, + "lock_path": "/tmp/tensorfold-model.lock", + "guard_source": { + "path": "guard.py", + "sha256": "3c4118b62c3797e1871032fcbbb1bf7d390f143c2e93ddf564b309e7f49a4664", + "origin": "Existing M5 narrow_guard exactaliases, addingknownrowtune/chunkbuilds; no admission rule weakened" + }, + "lock_and_completion_events": [ + { + "event": "lock_wait", + "time_ns": 1793213351158947000, + "path": "/tmp/tensorfold-model.lock" + }, + { + "event": "lock_acquired", + "time_ns": 1793213351159128000, + "path": "/tmp/tensorfold-model.lock", + "pid": 34921, + "device": 16777233, + "inode": 2622134 + }, + { + "event": "complete", + "time_ns": 1793213728179653000, + "legs": 16, + "all_lengths_validated": true, + "performance_gate_claim": false + }, + { + "event": "lock_release", + "time_ns": 1793213728179878000, + "pid": 34921 + } + ], + "owned_model_load_ready_cleanup_events": [ + { + "event": "owned_model_load_begin", + "time_ns": 1793213354237076000, + "tag": "text-6-0-current", + "pid": 34935, + "command": [ + "$RUN/bin/tensorfold-native-current", + "serve", + "$QUALIFICATION/LFM3-1.7B-MLX-6bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "event": "owned_model_ready", + "time_ns": 1793213355366020000, + "tag": "text-6-0-current", + "api_model": "lfm3-local", + "models": [ + { + "id": "lfm3-local", + "object": "model", + "created": 1793213355, + "owned_by": "tensorfold", + "context_length": 70000, + "max_model_len": 70000, + "reasoning": { + "default_enabled": false + } + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793213380272146000, + "tag": "text-6-0-current", + "pid": 34935, + "exit": 0, + "memory": { + "max_rss_bytes": 1647902720, + "peak_physical_footprint_bytes": 2063468512 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793213380391870000, + "tag": "text-6-1-standard", + "pid": 34989, + "command": [ + "$QUALIFICATION/standard-venv/bin/python", + "-m", + "mlx_vlm.server", + "--model", + "$QUALIFICATION/LFM3-1.7B-MLX-6bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--max-tokens", + "256", + "--max-num-seqs", + "4", + "--max-kv-size", + "70000", + "--vision-cache-size", + "0", + "--log-progress-interval", + "0" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": "70000" + }, + { + "event": "owned_model_ready", + "time_ns": 1793213383020576000, + "tag": "text-6-1-standard", + "api_model": "$QUALIFICATION/LFM3-1.7B-MLX-6bit", + "models": [ + { + "id": "$QUALIFICATION/LFM3-1.7B-MLX-6bit", + "object": "model", + "created": 1793213382, + "loaded": true + }, + { + "id": "Jundot/Qwen3.8-27B-oQ4e-mtp", + "object": "model", + "created": 1792174602, + "loaded": false + }, + { + "id": "Jundot/Qwen3.8-27B-oQ6e-mtp", + "object": "model", + "created": 1792175050, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-4bit", + "object": "model", + "created": 1792171219, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-6bit", + "object": "model", + "created": 1792174917, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-8bit", + "object": "model", + "created": 1792175293, + "loaded": false + }, + { + "id": "mlx-community/Qwen3.5-2B-MLX-4bit", + "object": "model", + "created": 1792174497, + "loaded": false + }, + { + "id": "TensorFold/GLM-5.3-Flash-MLX-4bit-MTP", + "object": "model", + "created": 1792171408, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-MLX-4bit", + "object": "model", + "created": 1791784422, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ4", + "object": "model", + "created": 1792174768, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ6", + "object": "model", + "created": 1792175440, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ8", + "object": "model", + "created": 1792175643, + "loaded": false + }, + { + "id": "TensorFold/Qwen3.8-27B-MLX-4bit", + "object": "model", + "created": 1792172626, + "loaded": false + }, + { + "id": "TensorFold/Qwen3.8-Flash-Next-MLX-6bit-MTP", + "object": "model", + "created": 1792176643, + "loaded": false + }, + { + "id": "Vontra/GLM-5.3-Flash-MLX-4bit-MTP", + "object": "model", + "created": 1790874074, + "loaded": false + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793213400591642000, + "tag": "text-6-1-standard", + "pid": 34989, + "exit": -15, + "memory": { + "max_rss_bytes": 1913864192, + "peak_physical_footprint_bytes": 3210103328 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793213400692696000, + "tag": "text-6-2-standard", + "pid": 35026, + "command": [ + "$QUALIFICATION/standard-venv/bin/python", + "-m", + "mlx_vlm.server", + "--model", + "$QUALIFICATION/LFM3-1.7B-MLX-6bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--max-tokens", + "256", + "--max-num-seqs", + "4", + "--max-kv-size", + "70000", + "--vision-cache-size", + "0", + "--log-progress-interval", + "0" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": "70000" + }, + { + "event": "owned_model_ready", + "time_ns": 1793213403296201000, + "tag": "text-6-2-standard", + "api_model": "$QUALIFICATION/LFM3-1.7B-MLX-6bit", + "models": [ + { + "id": "$QUALIFICATION/LFM3-1.7B-MLX-6bit", + "object": "model", + "created": 1793213402, + "loaded": true + }, + { + "id": "Jundot/Qwen3.8-27B-oQ4e-mtp", + "object": "model", + "created": 1792174602, + "loaded": false + }, + { + "id": "Jundot/Qwen3.8-27B-oQ6e-mtp", + "object": "model", + "created": 1792175050, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-4bit", + "object": "model", + "created": 1792171219, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-6bit", + "object": "model", + "created": 1792174917, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-8bit", + "object": "model", + "created": 1792175293, + "loaded": false + }, + { + "id": "mlx-community/Qwen3.5-2B-MLX-4bit", + "object": "model", + "created": 1792174497, + "loaded": false + }, + { + "id": "TensorFold/GLM-5.3-Flash-MLX-4bit-MTP", + "object": "model", + "created": 1792171408, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-MLX-4bit", + "object": "model", + "created": 1791784422, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ4", + "object": "model", + "created": 1792174768, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ6", + "object": "model", + "created": 1792175440, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ8", + "object": "model", + "created": 1792175643, + "loaded": false + }, + { + "id": "TensorFold/Qwen3.8-27B-MLX-4bit", + "object": "model", + "created": 1792172626, + "loaded": false + }, + { + "id": "TensorFold/Qwen3.8-Flash-Next-MLX-6bit-MTP", + "object": "model", + "created": 1792176643, + "loaded": false + }, + { + "id": "Vontra/GLM-5.3-Flash-MLX-4bit-MTP", + "object": "model", + "created": 1790874074, + "loaded": false + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793213420988480000, + "tag": "text-6-2-standard", + "pid": 35026, + "exit": -15, + "memory": { + "max_rss_bytes": 1910816768, + "peak_physical_footprint_bytes": 3207236104 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793213421092200000, + "tag": "text-6-3-current", + "pid": 35057, + "command": [ + "$RUN/bin/tensorfold-native-current", + "serve", + "$QUALIFICATION/LFM3-1.7B-MLX-6bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "event": "owned_model_ready", + "time_ns": 1793213422207497000, + "tag": "text-6-3-current", + "api_model": "lfm3-local", + "models": [ + { + "id": "lfm3-local", + "object": "model", + "created": 1793213422, + "owned_by": "tensorfold", + "context_length": 70000, + "max_model_len": 70000, + "reasoning": { + "default_enabled": false + } + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793213447228534000, + "tag": "text-6-3-current", + "pid": 35057, + "exit": 0, + "memory": { + "max_rss_bytes": 1648001024, + "peak_physical_footprint_bytes": 2063534048 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793213447340107000, + "tag": "text-8-0-current", + "pid": 35099, + "command": [ + "$RUN/bin/tensorfold-native-current", + "serve", + "$QUALIFICATION/LFM3-1.7B-MLX-8bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "event": "owned_model_ready", + "time_ns": 1793213448457781000, + "tag": "text-8-0-current", + "api_model": "lfm3-local", + "models": [ + { + "id": "lfm3-local", + "object": "model", + "created": 1793213448, + "owned_by": "tensorfold", + "context_length": 70000, + "max_model_len": 70000, + "reasoning": { + "default_enabled": false + } + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793213473412021000, + "tag": "text-8-0-current", + "pid": 35099, + "exit": 0, + "memory": { + "max_rss_bytes": 2080063488, + "peak_physical_footprint_bytes": 2495826248 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793213473518387000, + "tag": "text-8-1-standard", + "pid": 35130, + "command": [ + "$QUALIFICATION/standard-venv/bin/python", + "-m", + "mlx_vlm.server", + "--model", + "$QUALIFICATION/LFM3-1.7B-MLX-8bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--max-tokens", + "256", + "--max-num-seqs", + "4", + "--max-kv-size", + "70000", + "--vision-cache-size", + "0", + "--log-progress-interval", + "0" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": "70000" + }, + { + "event": "owned_model_ready", + "time_ns": 1793213476130842000, + "tag": "text-8-1-standard", + "api_model": "$QUALIFICATION/LFM3-1.7B-MLX-8bit", + "models": [ + { + "id": "$QUALIFICATION/LFM3-1.7B-MLX-8bit", + "object": "model", + "created": 1793213475, + "loaded": true + }, + { + "id": "Jundot/Qwen3.8-27B-oQ4e-mtp", + "object": "model", + "created": 1792174602, + "loaded": false + }, + { + "id": "Jundot/Qwen3.8-27B-oQ6e-mtp", + "object": "model", + "created": 1792175050, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-4bit", + "object": "model", + "created": 1792171219, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-6bit", + "object": "model", + "created": 1792174917, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-8bit", + "object": "model", + "created": 1792175293, + "loaded": false + }, + { + "id": "mlx-community/Qwen3.5-2B-MLX-4bit", + "object": "model", + "created": 1792174497, + "loaded": false + }, + { + "id": "TensorFold/GLM-5.3-Flash-MLX-4bit-MTP", + "object": "model", + "created": 1792171408, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-MLX-4bit", + "object": "model", + "created": 1791784422, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ4", + "object": "model", + "created": 1792174768, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ6", + "object": "model", + "created": 1792175440, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ8", + "object": "model", + "created": 1792175643, + "loaded": false + }, + { + "id": "TensorFold/Qwen3.8-27B-MLX-4bit", + "object": "model", + "created": 1792172626, + "loaded": false + }, + { + "id": "TensorFold/Qwen3.8-Flash-Next-MLX-6bit-MTP", + "object": "model", + "created": 1792176643, + "loaded": false + }, + { + "id": "Vontra/GLM-5.3-Flash-MLX-4bit-MTP", + "object": "model", + "created": 1790874074, + "loaded": false + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793213494373976000, + "tag": "text-8-1-standard", + "pid": 35130, + "exit": -15, + "memory": { + "max_rss_bytes": 2342780928, + "peak_physical_footprint_bytes": 3638283048 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793213494485018000, + "tag": "text-8-2-standard", + "pid": 35168, + "command": [ + "$QUALIFICATION/standard-venv/bin/python", + "-m", + "mlx_vlm.server", + "--model", + "$QUALIFICATION/LFM3-1.7B-MLX-8bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--max-tokens", + "256", + "--max-num-seqs", + "4", + "--max-kv-size", + "70000", + "--vision-cache-size", + "0", + "--log-progress-interval", + "0" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": "70000" + }, + { + "event": "owned_model_ready", + "time_ns": 1793213497095913000, + "tag": "text-8-2-standard", + "api_model": "$QUALIFICATION/LFM3-1.7B-MLX-8bit", + "models": [ + { + "id": "$QUALIFICATION/LFM3-1.7B-MLX-8bit", + "object": "model", + "created": 1793213496, + "loaded": true + }, + { + "id": "Jundot/Qwen3.8-27B-oQ4e-mtp", + "object": "model", + "created": 1792174602, + "loaded": false + }, + { + "id": "Jundot/Qwen3.8-27B-oQ6e-mtp", + "object": "model", + "created": 1792175050, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-4bit", + "object": "model", + "created": 1792171219, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-6bit", + "object": "model", + "created": 1792174917, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-8bit", + "object": "model", + "created": 1792175293, + "loaded": false + }, + { + "id": "mlx-community/Qwen3.5-2B-MLX-4bit", + "object": "model", + "created": 1792174497, + "loaded": false + }, + { + "id": "TensorFold/GLM-5.3-Flash-MLX-4bit-MTP", + "object": "model", + "created": 1792171408, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-MLX-4bit", + "object": "model", + "created": 1791784422, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ4", + "object": "model", + "created": 1792174768, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ6", + "object": "model", + "created": 1792175440, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ8", + "object": "model", + "created": 1792175643, + "loaded": false + }, + { + "id": "TensorFold/Qwen3.8-27B-MLX-4bit", + "object": "model", + "created": 1792172626, + "loaded": false + }, + { + "id": "TensorFold/Qwen3.8-Flash-Next-MLX-6bit-MTP", + "object": "model", + "created": 1792176643, + "loaded": false + }, + { + "id": "Vontra/GLM-5.3-Flash-MLX-4bit-MTP", + "object": "model", + "created": 1790874074, + "loaded": false + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793213515234422000, + "tag": "text-8-2-standard", + "pid": 35168, + "exit": -15, + "memory": { + "max_rss_bytes": 2342862848, + "peak_physical_footprint_bytes": 3638463032 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793213515344871000, + "tag": "text-8-3-current", + "pid": 35204, + "command": [ + "$RUN/bin/tensorfold-native-current", + "serve", + "$QUALIFICATION/LFM3-1.7B-MLX-8bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "event": "owned_model_ready", + "time_ns": 1793213516466674000, + "tag": "text-8-3-current", + "api_model": "lfm3-local", + "models": [ + { + "id": "lfm3-local", + "object": "model", + "created": 1793213516, + "owned_by": "tensorfold", + "context_length": 70000, + "max_model_len": 70000, + "reasoning": { + "default_enabled": false + } + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793213541300752000, + "tag": "text-8-3-current", + "pid": 35204, + "exit": 0, + "memory": { + "max_rss_bytes": 2080079872, + "peak_physical_footprint_bytes": 2495711512 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793213541410892000, + "tag": "vl-6-0-current", + "pid": 35245, + "command": [ + "$RUN/bin/tensorfold-native-current", + "serve", + "$QUALIFICATION/LFM3-VL-2B-MLX-6bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "event": "owned_model_ready", + "time_ns": 1793213542531060000, + "tag": "vl-6-0-current", + "api_model": "lfm3-local", + "models": [ + { + "id": "lfm3-local", + "object": "model", + "created": 1793213542, + "owned_by": "tensorfold", + "context_length": 70000, + "max_model_len": 70000, + "reasoning": { + "default_enabled": false + } + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793213567484392000, + "tag": "vl-6-0-current", + "pid": 35245, + "exit": 0, + "memory": { + "max_rss_bytes": 2026733568, + "peak_physical_footprint_bytes": 2483521888 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793213567589438000, + "tag": "vl-6-1-standard", + "pid": 35291, + "command": [ + "$QUALIFICATION/standard-venv/bin/python", + "-m", + "mlx_vlm.server", + "--model", + "$QUALIFICATION/LFM3-VL-2B-MLX-6bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--max-tokens", + "256", + "--max-num-seqs", + "4", + "--max-kv-size", + "70000", + "--vision-cache-size", + "0", + "--log-progress-interval", + "0" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": "70000" + }, + { + "event": "owned_model_ready", + "time_ns": 1793213570216135000, + "tag": "vl-6-1-standard", + "api_model": "$QUALIFICATION/LFM3-VL-2B-MLX-6bit", + "models": [ + { + "id": "$QUALIFICATION/LFM3-VL-2B-MLX-6bit", + "object": "model", + "created": 1793213569, + "loaded": true + }, + { + "id": "Jundot/Qwen3.8-27B-oQ4e-mtp", + "object": "model", + "created": 1792174602, + "loaded": false + }, + { + "id": "Jundot/Qwen3.8-27B-oQ6e-mtp", + "object": "model", + "created": 1792175050, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-4bit", + "object": "model", + "created": 1792171219, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-6bit", + "object": "model", + "created": 1792174917, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-8bit", + "object": "model", + "created": 1792175293, + "loaded": false + }, + { + "id": "mlx-community/Qwen3.5-2B-MLX-4bit", + "object": "model", + "created": 1792174497, + "loaded": false + }, + { + "id": "TensorFold/GLM-5.3-Flash-MLX-4bit-MTP", + "object": "model", + "created": 1792171408, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-MLX-4bit", + "object": "model", + "created": 1791784422, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ4", + "object": "model", + "created": 1792174768, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ6", + "object": "model", + "created": 1792175440, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ8", + "object": "model", + "created": 1792175643, + "loaded": false + }, + { + "id": "TensorFold/Qwen3.8-27B-MLX-4bit", + "object": "model", + "created": 1792172626, + "loaded": false + }, + { + "id": "TensorFold/Qwen3.8-Flash-Next-MLX-6bit-MTP", + "object": "model", + "created": 1792176643, + "loaded": false + }, + { + "id": "Vontra/GLM-5.3-Flash-MLX-4bit-MTP", + "object": "model", + "created": 1790874074, + "loaded": false + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793213587933036000, + "tag": "vl-6-1-standard", + "pid": 35291, + "exit": -15, + "memory": { + "max_rss_bytes": 2292514816, + "peak_physical_footprint_bytes": 3587885792 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793213588041372000, + "tag": "vl-6-2-standard", + "pid": 35324, + "command": [ + "$QUALIFICATION/standard-venv/bin/python", + "-m", + "mlx_vlm.server", + "--model", + "$QUALIFICATION/LFM3-VL-2B-MLX-6bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--max-tokens", + "256", + "--max-num-seqs", + "4", + "--max-kv-size", + "70000", + "--vision-cache-size", + "0", + "--log-progress-interval", + "0" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": "70000" + }, + { + "event": "owned_model_ready", + "time_ns": 1793213590658279000, + "tag": "vl-6-2-standard", + "api_model": "$QUALIFICATION/LFM3-VL-2B-MLX-6bit", + "models": [ + { + "id": "$QUALIFICATION/LFM3-VL-2B-MLX-6bit", + "object": "model", + "created": 1793213590, + "loaded": true + }, + { + "id": "Jundot/Qwen3.8-27B-oQ4e-mtp", + "object": "model", + "created": 1792174602, + "loaded": false + }, + { + "id": "Jundot/Qwen3.8-27B-oQ6e-mtp", + "object": "model", + "created": 1792175050, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-4bit", + "object": "model", + "created": 1792171219, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-6bit", + "object": "model", + "created": 1792174917, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-8bit", + "object": "model", + "created": 1792175293, + "loaded": false + }, + { + "id": "mlx-community/Qwen3.5-2B-MLX-4bit", + "object": "model", + "created": 1792174497, + "loaded": false + }, + { + "id": "TensorFold/GLM-5.3-Flash-MLX-4bit-MTP", + "object": "model", + "created": 1792171408, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-MLX-4bit", + "object": "model", + "created": 1791784422, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ4", + "object": "model", + "created": 1792174768, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ6", + "object": "model", + "created": 1792175440, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ8", + "object": "model", + "created": 1792175643, + "loaded": false + }, + { + "id": "TensorFold/Qwen3.8-27B-MLX-4bit", + "object": "model", + "created": 1792172626, + "loaded": false + }, + { + "id": "TensorFold/Qwen3.8-Flash-Next-MLX-6bit-MTP", + "object": "model", + "created": 1792176643, + "loaded": false + }, + { + "id": "Vontra/GLM-5.3-Flash-MLX-4bit-MTP", + "object": "model", + "created": 1790874074, + "loaded": false + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793213608390799000, + "tag": "vl-6-2-standard", + "pid": 35324, + "exit": -15, + "memory": { + "max_rss_bytes": 2292301824, + "peak_physical_footprint_bytes": 3590556504 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793213608504633000, + "tag": "vl-6-3-current", + "pid": 35360, + "command": [ + "$RUN/bin/tensorfold-native-current", + "serve", + "$QUALIFICATION/LFM3-VL-2B-MLX-6bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "event": "owned_model_ready", + "time_ns": 1793213609625867000, + "tag": "vl-6-3-current", + "api_model": "lfm3-local", + "models": [ + { + "id": "lfm3-local", + "object": "model", + "created": 1793213609, + "owned_by": "tensorfold", + "context_length": 70000, + "max_model_len": 70000, + "reasoning": { + "default_enabled": false + } + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793213634697489000, + "tag": "vl-6-3-current", + "pid": 35360, + "exit": 0, + "memory": { + "max_rss_bytes": 2026733568, + "peak_physical_footprint_bytes": 2483505480 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793213634815933000, + "tag": "vl-8-0-current", + "pid": 35396, + "command": [ + "$RUN/bin/tensorfold-native-current", + "serve", + "$QUALIFICATION/LFM3-VL-2B-MLX-8bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "event": "owned_model_ready", + "time_ns": 1793213635934806000, + "tag": "vl-8-0-current", + "api_model": "lfm3-local", + "models": [ + { + "id": "lfm3-local", + "object": "model", + "created": 1793213635, + "owned_by": "tensorfold", + "context_length": 70000, + "max_model_len": 70000, + "reasoning": { + "default_enabled": false + } + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793213660591689000, + "tag": "vl-8-0-current", + "pid": 35396, + "exit": 0, + "memory": { + "max_rss_bytes": 2458845184, + "peak_physical_footprint_bytes": 2915682920 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793213660697145000, + "tag": "vl-8-1-standard", + "pid": 35427, + "command": [ + "$QUALIFICATION/standard-venv/bin/python", + "-m", + "mlx_vlm.server", + "--model", + "$QUALIFICATION/LFM3-VL-2B-MLX-8bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--max-tokens", + "256", + "--max-num-seqs", + "4", + "--max-kv-size", + "70000", + "--vision-cache-size", + "0", + "--log-progress-interval", + "0" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": "70000" + }, + { + "event": "owned_model_ready", + "time_ns": 1793213663312095000, + "tag": "vl-8-1-standard", + "api_model": "$QUALIFICATION/LFM3-VL-2B-MLX-8bit", + "models": [ + { + "id": "$QUALIFICATION/LFM3-VL-2B-MLX-8bit", + "object": "model", + "created": 1793213662, + "loaded": true + }, + { + "id": "Jundot/Qwen3.8-27B-oQ4e-mtp", + "object": "model", + "created": 1792174602, + "loaded": false + }, + { + "id": "Jundot/Qwen3.8-27B-oQ6e-mtp", + "object": "model", + "created": 1792175050, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-4bit", + "object": "model", + "created": 1792171219, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-6bit", + "object": "model", + "created": 1792174917, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-8bit", + "object": "model", + "created": 1792175293, + "loaded": false + }, + { + "id": "mlx-community/Qwen3.5-2B-MLX-4bit", + "object": "model", + "created": 1792174497, + "loaded": false + }, + { + "id": "TensorFold/GLM-5.3-Flash-MLX-4bit-MTP", + "object": "model", + "created": 1792171408, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-MLX-4bit", + "object": "model", + "created": 1791784422, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ4", + "object": "model", + "created": 1792174768, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ6", + "object": "model", + "created": 1792175440, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ8", + "object": "model", + "created": 1792175643, + "loaded": false + }, + { + "id": "TensorFold/Qwen3.8-27B-MLX-4bit", + "object": "model", + "created": 1792172626, + "loaded": false + }, + { + "id": "TensorFold/Qwen3.8-Flash-Next-MLX-6bit-MTP", + "object": "model", + "created": 1792176643, + "loaded": false + }, + { + "id": "Vontra/GLM-5.3-Flash-MLX-4bit-MTP", + "object": "model", + "created": 1790874074, + "loaded": false + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793213681418135000, + "tag": "vl-8-1-standard", + "pid": 35427, + "exit": -15, + "memory": { + "max_rss_bytes": 2724462592, + "peak_physical_footprint_bytes": 4021373952 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793213681521503000, + "tag": "vl-8-2-standard", + "pid": 35465, + "command": [ + "$QUALIFICATION/standard-venv/bin/python", + "-m", + "mlx_vlm.server", + "--model", + "$QUALIFICATION/LFM3-VL-2B-MLX-8bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--max-tokens", + "256", + "--max-num-seqs", + "4", + "--max-kv-size", + "70000", + "--vision-cache-size", + "0", + "--log-progress-interval", + "0" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": "70000" + }, + { + "event": "owned_model_ready", + "time_ns": 1793213684139404000, + "tag": "vl-8-2-standard", + "api_model": "$QUALIFICATION/LFM3-VL-2B-MLX-8bit", + "models": [ + { + "id": "$QUALIFICATION/LFM3-VL-2B-MLX-8bit", + "object": "model", + "created": 1793213683, + "loaded": true + }, + { + "id": "Jundot/Qwen3.8-27B-oQ4e-mtp", + "object": "model", + "created": 1792174602, + "loaded": false + }, + { + "id": "Jundot/Qwen3.8-27B-oQ6e-mtp", + "object": "model", + "created": 1792175050, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-4bit", + "object": "model", + "created": 1792171219, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-6bit", + "object": "model", + "created": 1792174917, + "loaded": false + }, + { + "id": "LiquidAI/LFM2.5-24B-A2B-MLX-8bit", + "object": "model", + "created": 1792175293, + "loaded": false + }, + { + "id": "mlx-community/Qwen3.5-2B-MLX-4bit", + "object": "model", + "created": 1792174497, + "loaded": false + }, + { + "id": "TensorFold/GLM-5.3-Flash-MLX-4bit-MTP", + "object": "model", + "created": 1792171408, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-MLX-4bit", + "object": "model", + "created": 1791784422, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ4", + "object": "model", + "created": 1792174768, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ6", + "object": "model", + "created": 1792175440, + "loaded": false + }, + { + "id": "TensorFold/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-oQ8", + "object": "model", + "created": 1792175643, + "loaded": false + }, + { + "id": "TensorFold/Qwen3.8-27B-MLX-4bit", + "object": "model", + "created": 1792172626, + "loaded": false + }, + { + "id": "TensorFold/Qwen3.8-Flash-Next-MLX-6bit-MTP", + "object": "model", + "created": 1792176643, + "loaded": false + }, + { + "id": "Vontra/GLM-5.3-Flash-MLX-4bit-MTP", + "object": "model", + "created": 1790874074, + "loaded": false + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793213702225910000, + "tag": "vl-8-2-standard", + "pid": 35465, + "exit": -15, + "memory": { + "max_rss_bytes": 2724315136, + "peak_physical_footprint_bytes": 4021898192 + } + }, + { + "event": "owned_model_load_begin", + "time_ns": 1793213702327113000, + "tag": "vl-8-3-current", + "pid": 35502, + "command": [ + "$RUN/bin/tensorfold-native-current", + "serve", + "$QUALIFICATION/LFM3-VL-2B-MLX-8bit", + "--host", + "127.0.0.1", + "--port", + "18085", + "--name", + "lfm3-local", + "--context", + "70000", + "--parallel", + "4", + "--prompt-cache-gib", + "0", + "--no-thinking", + "--temperature", + "0", + "--no-update-check", + "--metrics-open" + ], + "environment_overrides": { + "APC_ENABLED": "0", + "APC_DISK_ENABLED": "0", + "HF_HUB_OFFLINE": "1", + "HF_HUB_DISABLE_TELEMETRY": "1", + "MLX_TRUST_REMOTE_CODE": "false" + }, + "removed_preload_or_draft_env_names": [], + "explicit_standard_MAX_KV_SIZE": null + }, + { + "event": "owned_model_ready", + "time_ns": 1793213703449778000, + "tag": "vl-8-3-current", + "api_model": "lfm3-local", + "models": [ + { + "id": "lfm3-local", + "object": "model", + "created": 1793213703, + "owned_by": "tensorfold", + "context_length": 70000, + "max_model_len": 70000, + "reasoning": { + "default_enabled": false + } + } + ] + }, + { + "event": "owned_cleanup_complete", + "time_ns": 1793213728107333000, + "tag": "vl-8-3-current", + "pid": 35502, + "exit": 0, + "memory": { + "max_rss_bytes": 2458812416, + "peak_physical_footprint_bytes": 2915699280 + } + } + ], + "admission": "Existing exclusive GPU flock and exact model-process admission; only owned descendants admitted during each measurement. Native admission and MLX limits unchanged. No unrelated owner stopped.", + "cleanup_exit_policy": "Eight native servers exited0; eight standard wrappers exited\u221215 after the existing runner intentionally SIGTERMed only owned descendants; admission checked no model remained.", + "actual_prompt_scope": "Published exact request bodies/official token arrays and pinned family tokenizers retained; usage counts verified. This packet does not add a new live/tokenize comparison.", + "memory": "Native admission unchanged; MLX memory and wired limits unmodified. /usr/bin/time -l captures physical footprint and RSS; available server metrics preserved. No allocator/pinned peak inferred from RSS." + }, + "results": { + "reported_summary": { + "text": { + "6": { + "current": { + "median_of_two_reported_leg_cell_medians_tps": { + "long-code-chat/1": 268.95, + "long-code-chat/4": 787.3, + "long-prose-chat/1": 268.7, + "long-prose-chat/4": 786.8 + }, + "per_leg_reported_cell_medians_and_drift": { + "long-code-chat/1": { + "legs": [ + 268.9, + 269.0 + ], + "second_over_first": 1.0003718854592787 + }, + "long-code-chat/4": { + "legs": [ + 786.8, + 787.8 + ], + "second_over_first": 1.0012709710218608 + }, + "long-prose-chat/1": { + "legs": [ + 268.5, + 268.9 + ], + "second_over_first": 1.001489757914339 + }, + "long-prose-chat/4": { + "legs": [ + 786.7, + 786.9 + ], + "second_over_first": 1.0002542265158254 + } + }, + "cold_median_ttft_s": { + "2048": 0.9490738336462528, + "8192": 4.988816916476935 + }, + "cold_leg_ttft_s": { + "2048": [ + 0.9490549168549478, + 0.9490927504375577 + ], + "8192": [ + 4.98626804118976, + 4.99136579176411 + ] + }, + "max_measured_physical_footprint_bytes": 2063534048 + }, + "standard": { + "median_of_two_reported_leg_cell_medians_tps": { + "long-code-chat/1": 283.95, + "long-code-chat/4": 920.6, + "long-prose-chat/1": 281.9, + "long-prose-chat/4": 922.45 + }, + "per_leg_reported_cell_medians_and_drift": { + "long-code-chat/1": { + "legs": [ + 286.7, + 281.2 + ], + "second_over_first": 0.980816184164632 + }, + "long-code-chat/4": { + "legs": [ + 920.7, + 920.5 + ], + "second_over_first": 0.9997827739763223 + }, + "long-prose-chat/1": { + "legs": [ + 282.4, + 281.4 + ], + "second_over_first": 0.9964589235127479 + }, + "long-prose-chat/4": { + "legs": [ + 922.4, + 922.5 + ], + "second_over_first": 1.0001084128360798 + } + }, + "cold_median_ttft_s": { + "2048": 0.1622776046860963, + "8192": 0.3932355416473001 + }, + "cold_leg_ttft_s": { + "2048": [ + 0.16382091725245118, + 0.16073429211974144 + ], + "8192": [ + 0.3932498749345541, + 0.39322120836004615 + ] + }, + "max_measured_physical_footprint_bytes": 3210103328 + } + }, + "8": { + "current": { + "median_of_two_reported_leg_cell_medians_tps": { + "long-code-chat/1": 257.95, + "long-code-chat/4": 797.6, + "long-prose-chat/1": 255.85, + "long-prose-chat/4": 792.8 + }, + "per_leg_reported_cell_medians_and_drift": { + "long-code-chat/1": { + "legs": [ + 257.9, + 258.0 + ], + "second_over_first": 1.000387747188833 + }, + "long-code-chat/4": { + "legs": [ + 797.7, + 797.5 + ], + "second_over_first": 0.9997492791776357 + }, + "long-prose-chat/1": { + "legs": [ + 255.2, + 256.5 + ], + "second_over_first": 1.0050940438871474 + }, + "long-prose-chat/4": { + "legs": [ + 792.6, + 793.0 + ], + "second_over_first": 1.000504668180671 + } + }, + "cold_median_ttft_s": { + "2048": 0.8529429994523525, + "8192": 4.612212021136656 + }, + "cold_leg_ttft_s": { + "2048": [ + 0.8527809581719339, + 0.8531050407327712 + ], + "8192": [ + 4.60699445893988, + 4.617429583333433 + ] + }, + "max_measured_physical_footprint_bytes": 2495826248 + }, + "standard": { + "median_of_two_reported_leg_cell_medians_tps": { + "long-code-chat/1": 267.15, + "long-code-chat/4": 921.0, + "long-prose-chat/1": 269.95, + "long-prose-chat/4": 921.55 + }, + "per_leg_reported_cell_medians_and_drift": { + "long-code-chat/1": { + "legs": [ + 267.0, + 267.3 + ], + "second_over_first": 1.001123595505618 + }, + "long-code-chat/4": { + "legs": [ + 921.0, + 921.0 + ], + "second_over_first": 1.0 + }, + "long-prose-chat/1": { + "legs": [ + 266.7, + 273.2 + ], + "second_over_first": 1.0243719535058118 + }, + "long-prose-chat/4": { + "legs": [ + 921.5, + 921.6 + ], + "second_over_first": 1.0001085187194791 + } + }, + "cold_median_ttft_s": { + "2048": 0.15217368747107685, + "8192": 0.38321862556040287 + }, + "cold_leg_ttft_s": { + "2048": [ + 0.1519840001128614, + 0.1523633748292923 + ], + "8192": [ + 0.3826749590225518, + 0.38376229209825397 + ] + }, + "max_measured_physical_footprint_bytes": 3638463032 + } + } + }, + "vl": { + "6": { + "current": { + "median_of_two_reported_leg_cell_medians_tps": { + "long-code-chat/1": 271.5, + "long-code-chat/4": 793.5, + "long-prose-chat/1": 268.15, + "long-prose-chat/4": 788.5 + }, + "per_leg_reported_cell_medians_and_drift": { + "long-code-chat/1": { + "legs": [ + 271.5, + 271.5 + ], + "second_over_first": 1.0 + }, + "long-code-chat/4": { + "legs": [ + 792.6, + 794.4 + ], + "second_over_first": 1.0022710068130203 + }, + "long-prose-chat/1": { + "legs": [ + 268.6, + 267.7 + ], + "second_over_first": 0.9966492926284437 + }, + "long-prose-chat/4": { + "legs": [ + 788.5, + 788.5 + ], + "second_over_first": 1.0 + } + }, + "cold_median_ttft_s": { + "2048": 0.958015417214483, + "8192": 4.987887290772051 + }, + "cold_leg_ttft_s": { + "2048": [ + 0.9579860842786729, + 0.9580447501502931 + ], + "8192": [ + 4.989488790743053, + 4.986285790801048 + ] + }, + "max_measured_physical_footprint_bytes": 2483521888 + }, + "standard": { + "median_of_two_reported_leg_cell_medians_tps": { + "long-code-chat/1": 284.0, + "long-code-chat/4": 920.5, + "long-prose-chat/1": 281.4, + "long-prose-chat/4": 922.3 + }, + "per_leg_reported_cell_medians_and_drift": { + "long-code-chat/1": { + "legs": [ + 286.0, + 282.0 + ], + "second_over_first": 0.986013986013986 + }, + "long-code-chat/4": { + "legs": [ + 920.7, + 920.3 + ], + "second_over_first": 0.9995655479526446 + }, + "long-prose-chat/1": { + "legs": [ + 280.9, + 281.9 + ], + "second_over_first": 1.003559985760057 + }, + "long-prose-chat/4": { + "legs": [ + 921.8, + 922.8 + ], + "second_over_first": 1.0010848340203948 + } + }, + "cold_median_ttft_s": { + "2048": 0.1677484381943941, + "8192": 0.3943172914441675 + }, + "cold_leg_ttft_s": { + "2048": [ + 0.16758570913225412, + 0.1679111672565341 + ], + "8192": [ + 0.39433749997988343, + 0.39429708290845156 + ] + }, + "max_measured_physical_footprint_bytes": 3590556504 + } + }, + "8": { + "current": { + "median_of_two_reported_leg_cell_medians_tps": { + "long-code-chat/1": 261.45, + "long-code-chat/4": 806.9000000000001, + "long-prose-chat/1": 261.95, + "long-prose-chat/4": 806.45 + }, + "per_leg_reported_cell_medians_and_drift": { + "long-code-chat/1": { + "legs": [ + 261.4, + 261.5 + ], + "second_over_first": 1.0003825554705432 + }, + "long-code-chat/4": { + "legs": [ + 805.7, + 808.1 + ], + "second_over_first": 1.0029787762194364 + }, + "long-prose-chat/1": { + "legs": [ + 261.5, + 262.4 + ], + "second_over_first": 1.0034416826003822 + }, + "long-prose-chat/4": { + "legs": [ + 805.9, + 807.0 + ], + "second_over_first": 1.0013649336145924 + } + }, + "cold_median_ttft_s": { + "2048": 0.8489160002209246, + "8192": 4.51366399996914 + }, + "cold_leg_ttft_s": { + "2048": [ + 0.848729083314538, + 0.8491029171273112 + ], + "8192": [ + 4.514424624852836, + 4.512903375085443 + ] + }, + "max_measured_physical_footprint_bytes": 2915699280 + }, + "standard": { + "median_of_two_reported_leg_cell_medians_tps": { + "long-code-chat/1": 272.70000000000005, + "long-code-chat/4": 921.0, + "long-prose-chat/1": 266.85, + "long-prose-chat/4": 919.7 + }, + "per_leg_reported_cell_medians_and_drift": { + "long-code-chat/1": { + "legs": [ + 272.6, + 272.8 + ], + "second_over_first": 1.0007336757153338 + }, + "long-code-chat/4": { + "legs": [ + 921.6, + 920.4 + ], + "second_over_first": 0.9986979166666666 + }, + "long-prose-chat/1": { + "legs": [ + 266.7, + 267.0 + ], + "second_over_first": 1.001124859392576 + }, + "long-prose-chat/4": { + "legs": [ + 919.4, + 920.0 + ], + "second_over_first": 1.000652599521427 + } + }, + "cold_median_ttft_s": { + "2048": 0.1677154789213091, + "8192": 0.3841822915710509 + }, + "cold_leg_ttft_s": { + "2048": [ + 0.16829495783895254, + 0.16713600000366569 + ], + "8192": [ + 0.38476558309048414, + 0.3835990000516176 + ] + }, + "max_measured_physical_footprint_bytes": 4021898192 + } + } + } + }, + "summary_method": "Median of two existing per-leg reported cell medians; all three per-leg raw repetitions/evidence are retained below. Cold median uses two mirrored server-leg samples.", + "legs": [ + { + "family": "text", + "bits": 6, + "leg": 0, + "engine": "current", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.9490549168549478, + "output_sha256": "6cb715bc54d1b0adbe24d918f8f147ac719d55d25abea6c65bc204eb159bebe8", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27", + "token_sha256": "b482722716a27c86c613d49d437feba1fe030ed496fb1141b0498ac017ee32c1" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.98626804118976, + "output_sha256": "63b1d289496206389642e98d3a58dffa5a753c8e5e32a1c40ced27643265c352", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0", + "token_sha256": "06efc9b74a3d0b1c7f8b7e69ae2da268963d81d498818ac39f495593a6c6da42" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1647902720, + "peak_physical_footprint_bytes": 2063468512 + }, + "tag": "text-6-0-current", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 268.9, + "per_stream_tps": 268.9, + "timing_valid": true, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 268.88099219948776, + "per_stream_tps": [ + 268.9 + ], + "makespan_s": 0.9718139171600342, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c826dd2b65f7d4a" + ], + "comparison_refs": [ + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + } + ] + }, + { + "aggregate_tps": 269.1038208492842, + "per_stream_tps": [ + 269.1 + ], + "makespan_s": 0.9710836671292782, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c826dd2b65f7d4a" + ], + "comparison_refs": [ + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + } + ] + }, + { + "aggregate_tps": 268.51277096180075, + "per_stream_tps": [ + 268.5 + ], + "makespan_s": 0.9733099159784615, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c826dd2b65f7d4a" + ], + "comparison_refs": [ + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 786.8, + "per_stream_tps": 202.2, + "timing_valid": true, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 786.7912152964943, + "per_stream_tps": [ + 203.9, + 207.7, + 196.7, + 200.2 + ], + "makespan_s": 1.3206207500770688, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c826dd2b65f7d4a", + "0c826dd2b65f7d4a", + "0c826dd2b65f7d4a", + "0c826dd2b65f7d4a" + ], + "comparison_refs": [ + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + }, + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + }, + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + }, + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + } + ] + }, + { + "aggregate_tps": 786.5960942792813, + "per_stream_tps": [ + 207.7, + 196.7, + 203.9, + 200.2 + ], + "makespan_s": 1.320863917004317, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c826dd2b65f7d4a", + "0c826dd2b65f7d4a", + "0c826dd2b65f7d4a", + "0c826dd2b65f7d4a" + ], + "comparison_refs": [ + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + }, + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + }, + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + }, + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + } + ] + }, + { + "aggregate_tps": 787.7752474739107, + "per_stream_tps": [ + 204.3, + 208.0, + 200.6, + 197.0 + ], + "makespan_s": 1.318489333614707, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c826dd2b65f7d4a", + "0c826dd2b65f7d4a", + "0c826dd2b65f7d4a", + "0c826dd2b65f7d4a" + ], + "comparison_refs": [ + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + }, + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + }, + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + }, + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 268.5, + "per_stream_tps": 268.5, + "timing_valid": true, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 267.797026968795, + "per_stream_tps": [ + 267.8 + ], + "makespan_s": 0.9758602087385952, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "096c4e95759af816" + ], + "comparison_refs": [ + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + } + ] + }, + { + "aggregate_tps": 268.8051256371799, + "per_stream_tps": [ + 268.8 + ], + "makespan_s": 0.9722310840152204, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "096c4e95759af816" + ], + "comparison_refs": [ + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + } + ] + }, + { + "aggregate_tps": 268.5311385150706, + "per_stream_tps": [ + 268.5 + ], + "makespan_s": 0.9732342502102256, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "096c4e95759af816" + ], + "comparison_refs": [ + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 786.7, + "per_stream_tps": 202.4, + "timing_valid": true, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 789.1579242687191, + "per_stream_tps": [ + 200.9, + 208.5, + 204.6, + 197.3 + ], + "makespan_s": 1.3168592918664217, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "096c4e95759af816", + "096c4e95759af816", + "096c4e95759af816", + "096c4e95759af816" + ], + "comparison_refs": [ + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + }, + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + }, + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + }, + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + } + ] + }, + { + "aggregate_tps": 786.6721540668058, + "per_stream_tps": [ + 207.8, + 204.0, + 200.3, + 196.7 + ], + "makespan_s": 1.3210327080450952, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "096c4e95759af816", + "096c4e95759af816", + "096c4e95759af816", + "096c4e95759af816" + ], + "comparison_refs": [ + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + }, + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + }, + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + }, + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + } + ] + }, + { + "aggregate_tps": 785.9673743346722, + "per_stream_tps": [ + 196.5, + 200.1, + 207.6, + 203.8 + ], + "makespan_s": 1.321932875085622, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "096c4e95759af816", + "096c4e95759af816", + "096c4e95759af816", + "096c4e95759af816" + ], + "comparison_refs": [ + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + }, + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + }, + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + }, + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + } + ] + } + ] + } + ] + }, + { + "family": "text", + "bits": 6, + "leg": 1, + "engine": "standard", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.16382091725245118, + "output_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "request_body_sha256": "d186e6168076ea392b899be28eb23440cdc2b8a2ba4c208dcb0729ad83d873fa", + "token_sha256": null + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.3932498749345541, + "output_sha256": "63b1d289496206389642e98d3a58dffa5a753c8e5e32a1c40ced27643265c352", + "request_body_sha256": "e08051c419693c025d7fbce09a821a08bd08bf712b254cce7047c3c09ff1a5eb", + "token_sha256": null + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "unverified if server omits token hashes; text digest is not substituted" + }, + "memory": { + "max_rss_bytes": 1913864192, + "peak_physical_footprint_bytes": 3210103328 + }, + "tag": "text-6-1-standard", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 286.7, + "per_stream_tps": 286.7, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 286.9251043856005, + "per_stream_tps": [ + 286.9 + ], + "makespan_s": 0.9075288330204785, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "95cfa5b83dc42bc9" + ], + "comparison_refs": [ + { + "concurrent": "80f3f3ef627d7207", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + } + ] + }, + { + "aggregate_tps": 286.6843816526364, + "per_stream_tps": [ + 286.7 + ], + "makespan_s": 0.9082843330688775, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "95cfa5b83dc42bc9" + ], + "comparison_refs": [ + { + "concurrent": "80f3f3ef627d7207", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + } + ] + }, + { + "aggregate_tps": 285.89404372110994, + "per_stream_tps": [ + 285.9 + ], + "makespan_s": 0.9107885826379061, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "95cfa5b83dc42bc9" + ], + "comparison_refs": [ + { + "concurrent": "80f3f3ef627d7207", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 920.7, + "per_stream_tps": 233.2, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 911.503135214735, + "per_stream_tps": [ + 229.7, + 232.1, + 232.1, + 232.1 + ], + "makespan_s": 1.1541472501121461, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "ce7480621f864d70", + "87ec47f1bb1fecea", + "87ec47f1bb1fecea", + "87ec47f1bb1fecea" + ], + "comparison_refs": [ + { + "concurrent": "941c5e009cd44c4e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + }, + { + "concurrent": "5e308b1934cc926e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + }, + { + "concurrent": "5e308b1934cc926e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + }, + { + "concurrent": "5e308b1934cc926e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + } + ] + }, + { + "aggregate_tps": 920.7406646888757, + "per_stream_tps": [ + 234.2, + 232.0, + 234.2, + 234.2 + ], + "makespan_s": 1.14224216574803, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "87ec47f1bb1fecea", + "ce7480621f864d70", + "87ec47f1bb1fecea", + "87ec47f1bb1fecea" + ], + "comparison_refs": [ + { + "concurrent": "5e308b1934cc926e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + }, + { + "concurrent": "941c5e009cd44c4e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + }, + { + "concurrent": "5e308b1934cc926e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + }, + { + "concurrent": "5e308b1934cc926e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + } + ] + }, + { + "aggregate_tps": 921.2703368300529, + "per_stream_tps": [ + 232.2, + 234.4, + 234.4, + 234.4 + ], + "makespan_s": 1.1413542916998267, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "ce7480621f864d70", + "87ec47f1bb1fecea", + "87ec47f1bb1fecea", + "87ec47f1bb1fecea" + ], + "comparison_refs": [ + { + "concurrent": "941c5e009cd44c4e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + }, + { + "concurrent": "5e308b1934cc926e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + }, + { + "concurrent": "5e308b1934cc926e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + }, + { + "concurrent": "5e308b1934cc926e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 282.4, + "per_stream_tps": 282.4, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 281.98163623662333, + "per_stream_tps": [ + 282.0 + ], + "makespan_s": 0.924834500066936, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "312e01d99328eead" + ], + "comparison_refs": [ + { + "concurrent": "61b9f6421abd73f2", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + } + ] + }, + { + "aggregate_tps": 282.44214987815354, + "per_stream_tps": [ + 282.4 + ], + "makespan_s": 0.9234476666897535, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "312e01d99328eead" + ], + "comparison_refs": [ + { + "concurrent": "61b9f6421abd73f2", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + } + ] + }, + { + "aggregate_tps": 282.72039307738567, + "per_stream_tps": [ + 282.7 + ], + "makespan_s": 0.9225772079080343, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "312e01d99328eead" + ], + "comparison_refs": [ + { + "concurrent": "61b9f6421abd73f2", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 922.4, + "per_stream_tps": 234.2, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 919.8010163556756, + "per_stream_tps": [ + 231.7, + 233.9, + 233.9, + 231.6 + ], + "makespan_s": 1.1559476656839252, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0bb42460b21e44af", + "4180ea3b03acb5e5", + "4180ea3b03acb5e5", + "0bb42460b21e44af" + ], + "comparison_refs": [ + { + "concurrent": "bec6286bf80c50f0", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + }, + { + "concurrent": "1f45bd085a7d2945", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + }, + { + "concurrent": "1f45bd085a7d2945", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + }, + { + "concurrent": "bec6286bf80c50f0", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + } + ] + }, + { + "aggregate_tps": 922.724325482371, + "per_stream_tps": [ + 232.5, + 234.8, + 234.8, + 234.8 + ], + "makespan_s": 1.1399587076157331, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "62099b9da36ffa99", + "354f62312ff31951", + "354f62312ff31951", + "354f62312ff31951" + ], + "comparison_refs": [ + { + "concurrent": "8cbc42429afcb8be", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + }, + { + "concurrent": "0de6688c1efa0d81", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + }, + { + "concurrent": "0de6688c1efa0d81", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + }, + { + "concurrent": "0de6688c1efa0d81", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + } + ] + }, + { + "aggregate_tps": 922.4092196565766, + "per_stream_tps": [ + 234.6, + 234.7, + 234.6, + 232.5 + ], + "makespan_s": 1.1401347918435931, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "354f62312ff31951", + "354f62312ff31951", + "354f62312ff31951", + "62099b9da36ffa99" + ], + "comparison_refs": [ + { + "concurrent": "0de6688c1efa0d81", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + }, + { + "concurrent": "0de6688c1efa0d81", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + }, + { + "concurrent": "0de6688c1efa0d81", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + }, + { + "concurrent": "8cbc42429afcb8be", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + } + ] + } + ] + } + ] + }, + { + "family": "text", + "bits": 6, + "leg": 2, + "engine": "standard", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.16073429211974144, + "output_sha256": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "request_body_sha256": "d186e6168076ea392b899be28eb23440cdc2b8a2ba4c208dcb0729ad83d873fa", + "token_sha256": null + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.39322120836004615, + "output_sha256": "63b1d289496206389642e98d3a58dffa5a753c8e5e32a1c40ced27643265c352", + "request_body_sha256": "e08051c419693c025d7fbce09a821a08bd08bf712b254cce7047c3c09ff1a5eb", + "token_sha256": null + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "unverified if server omits token hashes; text digest is not substituted" + }, + "memory": { + "max_rss_bytes": 1910816768, + "peak_physical_footprint_bytes": 3207236104 + }, + "tag": "text-6-2-standard", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 281.2, + "per_stream_tps": 281.2, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 280.5470979261015, + "per_stream_tps": [ + 280.5 + ], + "makespan_s": 0.9282331662252545, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "95cfa5b83dc42bc9" + ], + "comparison_refs": [ + { + "concurrent": "80f3f3ef627d7207", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + } + ] + }, + { + "aggregate_tps": 281.2324489124377, + "per_stream_tps": [ + 281.2 + ], + "makespan_s": 0.9273935840465128, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "95cfa5b83dc42bc9" + ], + "comparison_refs": [ + { + "concurrent": "80f3f3ef627d7207", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + } + ] + }, + { + "aggregate_tps": 282.01613529124614, + "per_stream_tps": [ + 282.0 + ], + "makespan_s": 0.9244711250066757, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "95cfa5b83dc42bc9" + ], + "comparison_refs": [ + { + "concurrent": "80f3f3ef627d7207", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 920.5, + "per_stream_tps": 234.0, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 917.7471375199799, + "per_stream_tps": [ + 231.3, + 233.8, + 233.8, + 233.8 + ], + "makespan_s": 1.1490790420211852, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "ce7480621f864d70", + "87ec47f1bb1fecea", + "87ec47f1bb1fecea", + "87ec47f1bb1fecea" + ], + "comparison_refs": [ + { + "concurrent": "941c5e009cd44c4e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + }, + { + "concurrent": "5e308b1934cc926e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + }, + { + "concurrent": "5e308b1934cc926e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + }, + { + "concurrent": "5e308b1934cc926e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + } + ] + }, + { + "aggregate_tps": 920.5722855818422, + "per_stream_tps": [ + 232.0, + 234.2, + 234.2, + 234.2 + ], + "makespan_s": 1.1422053333371878, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "ce7480621f864d70", + "87ec47f1bb1fecea", + "87ec47f1bb1fecea", + "87ec47f1bb1fecea" + ], + "comparison_refs": [ + { + "concurrent": "941c5e009cd44c4e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + }, + { + "concurrent": "5e308b1934cc926e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + }, + { + "concurrent": "5e308b1934cc926e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + }, + { + "concurrent": "5e308b1934cc926e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + } + ] + }, + { + "aggregate_tps": 920.4814218073618, + "per_stream_tps": [ + 234.2, + 234.2, + 234.2, + 231.9 + ], + "makespan_s": 1.1425432502292097, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "87ec47f1bb1fecea", + "87ec47f1bb1fecea", + "87ec47f1bb1fecea", + "ce7480621f864d70" + ], + "comparison_refs": [ + { + "concurrent": "5e308b1934cc926e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + }, + { + "concurrent": "5e308b1934cc926e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + }, + { + "concurrent": "5e308b1934cc926e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + }, + { + "concurrent": "941c5e009cd44c4e", + "alone": "80f3f3ef627d7207", + "serial": "80f3f3ef627d7207" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 281.4, + "per_stream_tps": 281.4, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 282.183617954872, + "per_stream_tps": [ + 282.2 + ], + "makespan_s": 0.9236649167723954, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "312e01d99328eead" + ], + "comparison_refs": [ + { + "concurrent": "61b9f6421abd73f2", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + } + ] + }, + { + "aggregate_tps": 281.3648087931368, + "per_stream_tps": [ + 281.4 + ], + "makespan_s": 0.926333624869585, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "312e01d99328eead" + ], + "comparison_refs": [ + { + "concurrent": "61b9f6421abd73f2", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + } + ] + }, + { + "aggregate_tps": 281.3050457351022, + "per_stream_tps": [ + 281.3 + ], + "makespan_s": 0.927065750118345, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "312e01d99328eead" + ], + "comparison_refs": [ + { + "concurrent": "61b9f6421abd73f2", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 922.5, + "per_stream_tps": 233.9, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 916.2948897187083, + "per_stream_tps": [ + 233.2, + 233.2, + 233.2, + 230.9 + ], + "makespan_s": 1.150288499891758, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "354f62312ff31951", + "354f62312ff31951", + "354f62312ff31951", + "62099b9da36ffa99" + ], + "comparison_refs": [ + { + "concurrent": "0de6688c1efa0d81", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + }, + { + "concurrent": "0de6688c1efa0d81", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + }, + { + "concurrent": "0de6688c1efa0d81", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + }, + { + "concurrent": "8cbc42429afcb8be", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + } + ] + }, + { + "aggregate_tps": 922.5357853206202, + "per_stream_tps": [ + 232.5, + 234.6, + 234.7, + 234.6 + ], + "makespan_s": 1.139938292093575, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "62099b9da36ffa99", + "354f62312ff31951", + "354f62312ff31951", + "354f62312ff31951" + ], + "comparison_refs": [ + { + "concurrent": "8cbc42429afcb8be", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + }, + { + "concurrent": "0de6688c1efa0d81", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + }, + { + "concurrent": "0de6688c1efa0d81", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + }, + { + "concurrent": "0de6688c1efa0d81", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + } + ] + }, + { + "aggregate_tps": 922.9511476950466, + "per_stream_tps": [ + 234.8, + 234.8, + 234.8, + 232.6 + ], + "makespan_s": 1.1392235839739442, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "354f62312ff31951", + "354f62312ff31951", + "354f62312ff31951", + "62099b9da36ffa99" + ], + "comparison_refs": [ + { + "concurrent": "0de6688c1efa0d81", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + }, + { + "concurrent": "0de6688c1efa0d81", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + }, + { + "concurrent": "0de6688c1efa0d81", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + }, + { + "concurrent": "8cbc42429afcb8be", + "alone": "61b9f6421abd73f2", + "serial": "61b9f6421abd73f2" + } + ] + } + ] + } + ] + }, + { + "family": "text", + "bits": 6, + "leg": 3, + "engine": "current", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.9490927504375577, + "output_sha256": "6cb715bc54d1b0adbe24d918f8f147ac719d55d25abea6c65bc204eb159bebe8", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27", + "token_sha256": "b482722716a27c86c613d49d437feba1fe030ed496fb1141b0498ac017ee32c1" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.99136579176411, + "output_sha256": "63b1d289496206389642e98d3a58dffa5a753c8e5e32a1c40ced27643265c352", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0", + "token_sha256": "06efc9b74a3d0b1c7f8b7e69ae2da268963d81d498818ac39f495593a6c6da42" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 1648001024, + "peak_physical_footprint_bytes": 2063534048 + }, + "tag": "text-6-3-current", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 269.0, + "per_stream_tps": 269.0, + "timing_valid": true, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 269.3099762527689, + "per_stream_tps": [ + 269.3 + ], + "makespan_s": 0.9702967079356313, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c826dd2b65f7d4a" + ], + "comparison_refs": [ + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + } + ] + }, + { + "aggregate_tps": 268.8825045334123, + "per_stream_tps": [ + 268.9 + ], + "makespan_s": 0.9718289999291301, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c826dd2b65f7d4a" + ], + "comparison_refs": [ + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + } + ] + }, + { + "aggregate_tps": 268.98995479368375, + "per_stream_tps": [ + 269.0 + ], + "makespan_s": 0.9714073333889246, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c826dd2b65f7d4a" + ], + "comparison_refs": [ + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 787.8, + "per_stream_tps": 202.4, + "timing_valid": true, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 787.7570962554237, + "per_stream_tps": [ + 200.5, + 204.2, + 208.0, + 196.9 + ], + "makespan_s": 1.318868207745254, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c826dd2b65f7d4a", + "0c826dd2b65f7d4a", + "0c826dd2b65f7d4a", + "0c826dd2b65f7d4a" + ], + "comparison_refs": [ + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + }, + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + }, + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + }, + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + } + ] + }, + { + "aggregate_tps": 787.1956958151153, + "per_stream_tps": [ + 204.0, + 200.3, + 207.8, + 196.8 + ], + "makespan_s": 1.319965542294085, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c826dd2b65f7d4a", + "0c826dd2b65f7d4a", + "0c826dd2b65f7d4a", + "0c826dd2b65f7d4a" + ], + "comparison_refs": [ + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + }, + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + }, + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + }, + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + } + ] + }, + { + "aggregate_tps": 788.9171575917495, + "per_stream_tps": [ + 197.2, + 208.3, + 200.8, + 204.5 + ], + "makespan_s": 1.3170637907460332, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c826dd2b65f7d4a", + "0c826dd2b65f7d4a", + "0c826dd2b65f7d4a", + "0c826dd2b65f7d4a" + ], + "comparison_refs": [ + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + }, + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + }, + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + }, + { + "concurrent": "83aca5c0d9d2501c", + "alone": "83aca5c0d9d2501c", + "serial": "83aca5c0d9d2501c" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 268.9, + "per_stream_tps": 268.9, + "timing_valid": true, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 269.04364075573244, + "per_stream_tps": [ + 269.0 + ], + "makespan_s": 0.9714015838690102, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "096c4e95759af816" + ], + "comparison_refs": [ + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + } + ] + }, + { + "aggregate_tps": 268.25137204631903, + "per_stream_tps": [ + 268.3 + ], + "makespan_s": 0.9741837922483683, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "096c4e95759af816" + ], + "comparison_refs": [ + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + } + ] + }, + { + "aggregate_tps": 268.94894730499294, + "per_stream_tps": [ + 268.9 + ], + "makespan_s": 0.971713624894619, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "096c4e95759af816" + ], + "comparison_refs": [ + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 786.9, + "per_stream_tps": 202.2, + "timing_valid": true, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 786.8927331989756, + "per_stream_tps": [ + 204.0, + 200.3, + 207.8, + 196.7 + ], + "makespan_s": 1.3205281672999263, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "096c4e95759af816", + "096c4e95759af816", + "096c4e95759af816", + "096c4e95759af816" + ], + "comparison_refs": [ + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + }, + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + }, + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + }, + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + } + ] + }, + { + "aggregate_tps": 786.5925806409052, + "per_stream_tps": [ + 200.3, + 196.7, + 204.0, + 207.8 + ], + "makespan_s": 1.320845709182322, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "096c4e95759af816", + "096c4e95759af816", + "096c4e95759af816", + "096c4e95759af816" + ], + "comparison_refs": [ + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + }, + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + }, + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + }, + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + } + ] + }, + { + "aggregate_tps": 787.0391369687771, + "per_stream_tps": [ + 200.3, + 196.8, + 207.9, + 204.0 + ], + "makespan_s": 1.3202753327786922, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "096c4e95759af816", + "096c4e95759af816", + "096c4e95759af816", + "096c4e95759af816" + ], + "comparison_refs": [ + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + }, + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + }, + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + }, + { + "concurrent": "b11dff03a9d60648", + "alone": "b11dff03a9d60648", + "serial": "b11dff03a9d60648" + } + ] + } + ] + } + ] + }, + { + "family": "text", + "bits": 8, + "leg": 0, + "engine": "current", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.8527809581719339, + "output_sha256": "63b1d289496206389642e98d3a58dffa5a753c8e5e32a1c40ced27643265c352", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27", + "token_sha256": "06efc9b74a3d0b1c7f8b7e69ae2da268963d81d498818ac39f495593a6c6da42" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.60699445893988, + "output_sha256": "f6fe0f139e7e212c4558b810f2bc93e49dbe1b63c81c9c14759f5eaf0e3952a6", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0", + "token_sha256": "711bccf584b7cba05cf72e92b63640c9025fc1ba4468767322fe65278e9b91bd" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 2080063488, + "peak_physical_footprint_bytes": 2495826248 + }, + "tag": "text-8-0-current", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 257.9, + "per_stream_tps": 257.9, + "timing_valid": true, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 257.8923810893886, + "per_stream_tps": [ + 257.9 + ], + "makespan_s": 1.009470249991864, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "2b2419c32c37f85c" + ], + "comparison_refs": [ + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + } + ] + }, + { + "aggregate_tps": 258.1183168140391, + "per_stream_tps": [ + 258.1 + ], + "makespan_s": 1.0085951248183846, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "2b2419c32c37f85c" + ], + "comparison_refs": [ + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + } + ] + }, + { + "aggregate_tps": 257.5807119881568, + "per_stream_tps": [ + 257.6 + ], + "makespan_s": 1.0106695420108736, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "2b2419c32c37f85c" + ], + "comparison_refs": [ + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 797.7, + "per_stream_tps": 204.2, + "timing_valid": true, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 797.6789629295251, + "per_stream_tps": [ + 202.6, + 205.9, + 209.4, + 199.4 + ], + "makespan_s": 1.2999728326685727, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "2b2419c32c37f85c", + "2b2419c32c37f85c", + "2b2419c32c37f85c", + "2b2419c32c37f85c" + ], + "comparison_refs": [ + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + }, + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + }, + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + }, + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + } + ] + }, + { + "aggregate_tps": 798.3762956212927, + "per_stream_tps": [ + 206.1, + 202.8, + 209.5, + 199.6 + ], + "makespan_s": 1.299070707987994, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "2b2419c32c37f85c", + "2b2419c32c37f85c", + "2b2419c32c37f85c", + "2b2419c32c37f85c" + ], + "comparison_refs": [ + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + }, + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + }, + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + }, + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + } + ] + }, + { + "aggregate_tps": 796.5812090151402, + "per_stream_tps": [ + 209.0, + 199.2, + 205.7, + 202.4 + ], + "makespan_s": 1.3017636248841882, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "2b2419c32c37f85c", + "2b2419c32c37f85c", + "2b2419c32c37f85c", + "2b2419c32c37f85c" + ], + "comparison_refs": [ + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + }, + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + }, + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + }, + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 255.2, + "per_stream_tps": 255.2, + "timing_valid": true, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 255.15586833233786, + "per_stream_tps": [ + 255.2 + ], + "makespan_s": 1.0204067081212997, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "bf04e149adfd45bd" + ], + "comparison_refs": [ + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + } + ] + }, + { + "aggregate_tps": 255.1390507127601, + "per_stream_tps": [ + 255.1 + ], + "makespan_s": 1.0203968328423798, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "bf04e149adfd45bd" + ], + "comparison_refs": [ + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + } + ] + }, + { + "aggregate_tps": 255.30874284454575, + "per_stream_tps": [ + 255.3 + ], + "makespan_s": 1.0196791249327362, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "bf04e149adfd45bd" + ], + "comparison_refs": [ + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 792.6, + "per_stream_tps": 203.2, + "timing_valid": true, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 792.6027883949107, + "per_stream_tps": [ + 208.1, + 201.4, + 204.7, + 198.2 + ], + "makespan_s": 1.3083278751000762, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "bf04e149adfd45bd", + "bf04e149adfd45bd", + "bf04e149adfd45bd", + "bf04e149adfd45bd" + ], + "comparison_refs": [ + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + }, + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + }, + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + }, + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + } + ] + }, + { + "aggregate_tps": 793.794756442077, + "per_stream_tps": [ + 198.5, + 201.7, + 208.4, + 205.0 + ], + "makespan_s": 1.3064143750816584, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "bf04e149adfd45bd", + "bf04e149adfd45bd", + "bf04e149adfd45bd", + "bf04e149adfd45bd" + ], + "comparison_refs": [ + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + }, + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + }, + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + }, + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + } + ] + }, + { + "aggregate_tps": 792.5378676287996, + "per_stream_tps": [ + 204.7, + 201.3, + 198.1, + 208.1 + ], + "makespan_s": 1.3085306249558926, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "bf04e149adfd45bd", + "bf04e149adfd45bd", + "bf04e149adfd45bd", + "bf04e149adfd45bd" + ], + "comparison_refs": [ + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + }, + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + }, + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + }, + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + } + ] + } + ] + } + ] + }, + { + "family": "text", + "bits": 8, + "leg": 1, + "engine": "standard", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.1519840001128614, + "output_sha256": "63b1d289496206389642e98d3a58dffa5a753c8e5e32a1c40ced27643265c352", + "request_body_sha256": "3a2d7ca2d1393e41c8164cbba4349b09bf754d9fd932c6b6489c661a36c8c42e", + "token_sha256": null + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.3826749590225518, + "output_sha256": "f6fe0f139e7e212c4558b810f2bc93e49dbe1b63c81c9c14759f5eaf0e3952a6", + "request_body_sha256": "c87aa1143fa7a4d03e9fecde8a8b0a2d5ea5a4e1b71c98e0a557ad514d6a1784", + "token_sha256": null + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "unverified if server omits token hashes; text digest is not substituted" + }, + "memory": { + "max_rss_bytes": 2342780928, + "peak_physical_footprint_bytes": 3638283048 + }, + "tag": "text-8-1-standard", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 267.0, + "per_stream_tps": 267.0, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 267.12905011907, + "per_stream_tps": [ + 267.1 + ], + "makespan_s": 0.9756173337809741, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "6b9813fdb6f1119f" + ], + "comparison_refs": [ + { + "concurrent": "7fb3555bf55069a6", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + } + ] + }, + { + "aggregate_tps": 266.68852471525264, + "per_stream_tps": [ + 266.7 + ], + "makespan_s": 0.9771621250547469, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "6b9813fdb6f1119f" + ], + "comparison_refs": [ + { + "concurrent": "7fb3555bf55069a6", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + } + ] + }, + { + "aggregate_tps": 266.9845238934097, + "per_stream_tps": [ + 267.0 + ], + "makespan_s": 0.976414917036891, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "6b9813fdb6f1119f" + ], + "comparison_refs": [ + { + "concurrent": "7fb3555bf55069a6", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 921.0, + "per_stream_tps": 234.2, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 918.2186392073971, + "per_stream_tps": [ + 234.0, + 234.0, + 233.8, + 231.4 + ], + "makespan_s": 1.1486684996634722, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a3b4229cb8c41277", + "a3b4229cb8c41277", + "a3b4229cb8c41277", + "4e2ce2be59af5602" + ], + "comparison_refs": [ + { + "concurrent": "53a7029fb9ac15d0", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + }, + { + "concurrent": "53a7029fb9ac15d0", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + }, + { + "concurrent": "53a7029fb9ac15d0", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + }, + { + "concurrent": "260f5a902140f8fe", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + } + ] + }, + { + "aggregate_tps": 920.9707065891416, + "per_stream_tps": [ + 234.3, + 234.3, + 234.3, + 232.1 + ], + "makespan_s": 1.1420391662977636, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a3b4229cb8c41277", + "a3b4229cb8c41277", + "a3b4229cb8c41277", + "4e2ce2be59af5602" + ], + "comparison_refs": [ + { + "concurrent": "53a7029fb9ac15d0", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + }, + { + "concurrent": "53a7029fb9ac15d0", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + }, + { + "concurrent": "53a7029fb9ac15d0", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + }, + { + "concurrent": "260f5a902140f8fe", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + } + ] + }, + { + "aggregate_tps": 922.3649414718765, + "per_stream_tps": [ + 234.7, + 234.7, + 232.5, + 234.7 + ], + "makespan_s": 1.1402067500166595, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a3b4229cb8c41277", + "a3b4229cb8c41277", + "4e2ce2be59af5602", + "a3b4229cb8c41277" + ], + "comparison_refs": [ + { + "concurrent": "53a7029fb9ac15d0", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + }, + { + "concurrent": "53a7029fb9ac15d0", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + }, + { + "concurrent": "260f5a902140f8fe", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + }, + { + "concurrent": "53a7029fb9ac15d0", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 266.7, + "per_stream_tps": 266.7, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 266.1874597125077, + "per_stream_tps": [ + 266.2 + ], + "makespan_s": 0.976956166792661, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "9c6e8cc4553be330" + ], + "comparison_refs": [ + { + "concurrent": "47d6debc7dedd30c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + } + ] + }, + { + "aggregate_tps": 267.09328266486307, + "per_stream_tps": [ + 267.1 + ], + "makespan_s": 0.9754301249049604, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "9c6e8cc4553be330" + ], + "comparison_refs": [ + { + "concurrent": "47d6debc7dedd30c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + } + ] + }, + { + "aggregate_tps": 266.6992749656241, + "per_stream_tps": [ + 266.7 + ], + "makespan_s": 0.9776119580492377, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "9c6e8cc4553be330" + ], + "comparison_refs": [ + { + "concurrent": "47d6debc7dedd30c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 921.5, + "per_stream_tps": 234.2, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 918.7648948193362, + "per_stream_tps": [ + 234.0, + 234.0, + 234.0, + 231.5 + ], + "makespan_s": 1.147554833907634, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "c7bc7fd07326540b", + "c7bc7fd07326540b", + "c7bc7fd07326540b", + "c7bc7fd07326540b" + ], + "comparison_refs": [ + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + }, + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + }, + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + }, + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + } + ] + }, + { + "aggregate_tps": 922.2621532530628, + "per_stream_tps": [ + 232.4, + 234.7, + 234.6, + 234.7 + ], + "makespan_s": 1.1407705000601709, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "c7bc7fd07326540b", + "c7bc7fd07326540b", + "c7bc7fd07326540b", + "c7bc7fd07326540b" + ], + "comparison_refs": [ + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + }, + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + }, + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + }, + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + } + ] + }, + { + "aggregate_tps": 921.5008854164033, + "per_stream_tps": [ + 234.4, + 234.4, + 232.2, + 234.4 + ], + "makespan_s": 1.1412312500178814, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "c7bc7fd07326540b", + "c7bc7fd07326540b", + "c7bc7fd07326540b", + "c7bc7fd07326540b" + ], + "comparison_refs": [ + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + }, + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + }, + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + }, + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + } + ] + } + ] + } + ] + }, + { + "family": "text", + "bits": 8, + "leg": 2, + "engine": "standard", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.1523633748292923, + "output_sha256": "63b1d289496206389642e98d3a58dffa5a753c8e5e32a1c40ced27643265c352", + "request_body_sha256": "3a2d7ca2d1393e41c8164cbba4349b09bf754d9fd932c6b6489c661a36c8c42e", + "token_sha256": null + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.38376229209825397, + "output_sha256": "f6fe0f139e7e212c4558b810f2bc93e49dbe1b63c81c9c14759f5eaf0e3952a6", + "request_body_sha256": "c87aa1143fa7a4d03e9fecde8a8b0a2d5ea5a4e1b71c98e0a557ad514d6a1784", + "token_sha256": null + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "unverified if server omits token hashes; text digest is not substituted" + }, + "memory": { + "max_rss_bytes": 2342862848, + "peak_physical_footprint_bytes": 3638463032 + }, + "tag": "text-8-2-standard", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 267.3, + "per_stream_tps": 267.3, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 267.25612089747017, + "per_stream_tps": [ + 267.3 + ], + "makespan_s": 0.9756779577583075, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "6b9813fdb6f1119f" + ], + "comparison_refs": [ + { + "concurrent": "7fb3555bf55069a6", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + } + ] + }, + { + "aggregate_tps": 268.09450654845926, + "per_stream_tps": [ + 268.1 + ], + "makespan_s": 0.972224208060652, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "6b9813fdb6f1119f" + ], + "comparison_refs": [ + { + "concurrent": "7fb3555bf55069a6", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + } + ] + }, + { + "aggregate_tps": 267.3467641480941, + "per_stream_tps": [ + 267.3 + ], + "makespan_s": 0.9747204999439418, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "6b9813fdb6f1119f" + ], + "comparison_refs": [ + { + "concurrent": "7fb3555bf55069a6", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 921.0, + "per_stream_tps": 233.9, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 916.0495252048985, + "per_stream_tps": [ + 233.5, + 230.9, + 233.4, + 233.4 + ], + "makespan_s": 1.1516642081551254, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a3b4229cb8c41277", + "4e2ce2be59af5602", + "a3b4229cb8c41277", + "a3b4229cb8c41277" + ], + "comparison_refs": [ + { + "concurrent": "53a7029fb9ac15d0", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + }, + { + "concurrent": "260f5a902140f8fe", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + }, + { + "concurrent": "53a7029fb9ac15d0", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + }, + { + "concurrent": "53a7029fb9ac15d0", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + } + ] + }, + { + "aggregate_tps": 921.0122858045834, + "per_stream_tps": [ + 234.3, + 234.3, + 234.3, + 232.1 + ], + "makespan_s": 1.1419378342106938, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a3b4229cb8c41277", + "a3b4229cb8c41277", + "a3b4229cb8c41277", + "4e2ce2be59af5602" + ], + "comparison_refs": [ + { + "concurrent": "53a7029fb9ac15d0", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + }, + { + "concurrent": "53a7029fb9ac15d0", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + }, + { + "concurrent": "53a7029fb9ac15d0", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + }, + { + "concurrent": "260f5a902140f8fe", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + } + ] + }, + { + "aggregate_tps": 923.8205428055306, + "per_stream_tps": [ + 235.0, + 232.8, + 235.0, + 235.0 + ], + "makespan_s": 1.1386744170449674, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a3b4229cb8c41277", + "4e2ce2be59af5602", + "a3b4229cb8c41277", + "a3b4229cb8c41277" + ], + "comparison_refs": [ + { + "concurrent": "53a7029fb9ac15d0", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + }, + { + "concurrent": "260f5a902140f8fe", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + }, + { + "concurrent": "53a7029fb9ac15d0", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + }, + { + "concurrent": "53a7029fb9ac15d0", + "alone": "7fb3555bf55069a6", + "serial": "7fb3555bf55069a6" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 273.2, + "per_stream_tps": 273.2, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 273.3244940258848, + "per_stream_tps": [ + 273.3 + ], + "makespan_s": 0.9522101250477135, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "9c6e8cc4553be330" + ], + "comparison_refs": [ + { + "concurrent": "47d6debc7dedd30c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + } + ] + }, + { + "aggregate_tps": 273.0584350248402, + "per_stream_tps": [ + 273.1 + ], + "makespan_s": 0.9531893329694867, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "9c6e8cc4553be330" + ], + "comparison_refs": [ + { + "concurrent": "47d6debc7dedd30c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + } + ] + }, + { + "aggregate_tps": 273.1826989297863, + "per_stream_tps": [ + 273.2 + ], + "makespan_s": 0.9525825842283666, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "9c6e8cc4553be330" + ], + "comparison_refs": [ + { + "concurrent": "47d6debc7dedd30c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 921.6, + "per_stream_tps": 233.7, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 914.7279956009703, + "per_stream_tps": [ + 232.8, + 232.9, + 230.5, + 232.8 + ], + "makespan_s": 1.1492737093940377, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "c7bc7fd07326540b", + "c7bc7fd07326540b", + "c7bc7fd07326540b", + "c7bc7fd07326540b" + ], + "comparison_refs": [ + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + }, + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + }, + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + }, + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + } + ] + }, + { + "aggregate_tps": 921.6499645222126, + "per_stream_tps": [ + 234.5, + 234.5, + 234.5, + 232.3 + ], + "makespan_s": 1.1413378329016268, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "c7bc7fd07326540b", + "c7bc7fd07326540b", + "c7bc7fd07326540b", + "c7bc7fd07326540b" + ], + "comparison_refs": [ + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + }, + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + }, + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + }, + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + } + ] + }, + { + "aggregate_tps": 921.7208255094804, + "per_stream_tps": [ + 234.5, + 234.6, + 234.6, + 232.3 + ], + "makespan_s": 1.1409873329102993, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "c7bc7fd07326540b", + "c7bc7fd07326540b", + "c7bc7fd07326540b", + "c7bc7fd07326540b" + ], + "comparison_refs": [ + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + }, + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + }, + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + }, + { + "concurrent": "5898795258a4c16c", + "alone": "47d6debc7dedd30c", + "serial": "47d6debc7dedd30c" + } + ] + } + ] + } + ] + }, + { + "family": "text", + "bits": 8, + "leg": 3, + "engine": "current", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.8531050407327712, + "output_sha256": "63b1d289496206389642e98d3a58dffa5a753c8e5e32a1c40ced27643265c352", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27", + "token_sha256": "06efc9b74a3d0b1c7f8b7e69ae2da268963d81d498818ac39f495593a6c6da42" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.617429583333433, + "output_sha256": "f6fe0f139e7e212c4558b810f2bc93e49dbe1b63c81c9c14759f5eaf0e3952a6", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0", + "token_sha256": "711bccf584b7cba05cf72e92b63640c9025fc1ba4468767322fe65278e9b91bd" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 2080079872, + "peak_physical_footprint_bytes": 2495711512 + }, + "tag": "text-8-3-current", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 258.0, + "per_stream_tps": 258.0, + "timing_valid": true, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 257.97594595944804, + "per_stream_tps": [ + 258.0 + ], + "makespan_s": 1.0092443749308586, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "2b2419c32c37f85c" + ], + "comparison_refs": [ + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + } + ] + }, + { + "aggregate_tps": 258.2245577547794, + "per_stream_tps": [ + 258.2 + ], + "makespan_s": 1.0082863327115774, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "2b2419c32c37f85c" + ], + "comparison_refs": [ + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + } + ] + }, + { + "aggregate_tps": 257.6318489251904, + "per_stream_tps": [ + 257.6 + ], + "makespan_s": 1.0104822078719735, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "2b2419c32c37f85c" + ], + "comparison_refs": [ + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 797.5, + "per_stream_tps": 204.2, + "timing_valid": true, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 796.9964435593527, + "per_stream_tps": [ + 202.5, + 205.8, + 209.2, + 199.3 + ], + "makespan_s": 1.3008707500994205, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "2b2419c32c37f85c", + "2b2419c32c37f85c", + "2b2419c32c37f85c", + "2b2419c32c37f85c" + ], + "comparison_refs": [ + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + }, + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + }, + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + }, + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + } + ] + }, + { + "aggregate_tps": 797.9578276814573, + "per_stream_tps": [ + 206.0, + 199.5, + 209.4, + 202.7 + ], + "makespan_s": 1.2997064581140876, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "2b2419c32c37f85c", + "2b2419c32c37f85c", + "2b2419c32c37f85c", + "2b2419c32c37f85c" + ], + "comparison_refs": [ + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + }, + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + }, + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + }, + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + } + ] + }, + { + "aggregate_tps": 797.4858343072125, + "per_stream_tps": [ + 199.4, + 209.3, + 202.6, + 205.9 + ], + "makespan_s": 1.3005292913876474, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "2b2419c32c37f85c", + "2b2419c32c37f85c", + "2b2419c32c37f85c", + "2b2419c32c37f85c" + ], + "comparison_refs": [ + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + }, + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + }, + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + }, + { + "concurrent": "0f6bb87696635e5a", + "alone": "0f6bb87696635e5a", + "serial": "0f6bb87696635e5a" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 256.5, + "per_stream_tps": 256.5, + "timing_valid": true, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 256.0650814537741, + "per_stream_tps": [ + 256.1 + ], + "makespan_s": 1.016779042314738, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "bf04e149adfd45bd" + ], + "comparison_refs": [ + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + } + ] + }, + { + "aggregate_tps": 256.47663218991704, + "per_stream_tps": [ + 256.5 + ], + "makespan_s": 1.015176416374743, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "bf04e149adfd45bd" + ], + "comparison_refs": [ + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + } + ] + }, + { + "aggregate_tps": 256.5524847963025, + "per_stream_tps": [ + 256.6 + ], + "makespan_s": 1.0148482909426093, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "bf04e149adfd45bd" + ], + "comparison_refs": [ + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 793.0, + "per_stream_tps": 203.2, + "timing_valid": true, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 792.9833926672031, + "per_stream_tps": [ + 198.3, + 204.8, + 208.2, + 201.5 + ], + "makespan_s": 1.3075062921270728, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "bf04e149adfd45bd", + "bf04e149adfd45bd", + "bf04e149adfd45bd", + "bf04e149adfd45bd" + ], + "comparison_refs": [ + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + }, + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + }, + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + }, + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + } + ] + }, + { + "aggregate_tps": 794.019604968735, + "per_stream_tps": [ + 205.0, + 198.5, + 208.4, + 201.7 + ], + "makespan_s": 1.3062412082217634, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "bf04e149adfd45bd", + "bf04e149adfd45bd", + "bf04e149adfd45bd", + "bf04e149adfd45bd" + ], + "comparison_refs": [ + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + }, + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + }, + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + }, + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + } + ] + }, + { + "aggregate_tps": 792.997957490766, + "per_stream_tps": [ + 201.5, + 208.2, + 204.8, + 198.3 + ], + "makespan_s": 1.3079067920334637, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "bf04e149adfd45bd", + "bf04e149adfd45bd", + "bf04e149adfd45bd", + "bf04e149adfd45bd" + ], + "comparison_refs": [ + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + }, + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + }, + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + }, + { + "concurrent": "370dacf7affd1747", + "alone": "370dacf7affd1747", + "serial": "370dacf7affd1747" + } + ] + } + ] + } + ] + }, + { + "family": "vl", + "bits": 6, + "leg": 0, + "engine": "current", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.9579860842786729, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27", + "token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.989488790743053, + "output_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0", + "token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 2026733568, + "peak_physical_footprint_bytes": 2483521888 + }, + "tag": "vl-6-0-current", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 271.5, + "per_stream_tps": 271.5, + "timing_valid": true, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 271.1235719224361, + "per_stream_tps": [ + 271.1 + ], + "makespan_s": 0.963977915700525, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "ff5c2a57d0d6ba68" + ], + "comparison_refs": [ + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + } + ] + }, + { + "aggregate_tps": 271.49635622677846, + "per_stream_tps": [ + 271.5 + ], + "makespan_s": 0.9626684999093413, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "ff5c2a57d0d6ba68" + ], + "comparison_refs": [ + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + } + ] + }, + { + "aggregate_tps": 272.6169867060672, + "per_stream_tps": [ + 272.6 + ], + "makespan_s": 0.9588198750279844, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "ff5c2a57d0d6ba68" + ], + "comparison_refs": [ + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 792.6, + "per_stream_tps": 203.7, + "timing_valid": true, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 791.7046172222211, + "per_stream_tps": [ + 198.0, + 205.3, + 209.1, + 201.6 + ], + "makespan_s": 1.3122538751922548, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "ff5c2a57d0d6ba68", + "ff5c2a57d0d6ba68", + "ff5c2a57d0d6ba68", + "ff5c2a57d0d6ba68" + ], + "comparison_refs": [ + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + }, + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + }, + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + }, + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + } + ] + }, + { + "aggregate_tps": 792.5792051368239, + "per_stream_tps": [ + 198.1, + 201.7, + 205.5, + 209.3 + ], + "makespan_s": 1.31116933375597, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "ff5c2a57d0d6ba68", + "ff5c2a57d0d6ba68", + "ff5c2a57d0d6ba68", + "ff5c2a57d0d6ba68" + ], + "comparison_refs": [ + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + }, + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + }, + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + }, + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + } + ] + }, + { + "aggregate_tps": 793.9178369988279, + "per_stream_tps": [ + 205.8, + 198.5, + 209.7, + 202.1 + ], + "makespan_s": 1.3091860418207943, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "ff5c2a57d0d6ba68", + "ff5c2a57d0d6ba68", + "ff5c2a57d0d6ba68", + "ff5c2a57d0d6ba68" + ], + "comparison_refs": [ + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + }, + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + }, + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + }, + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 268.6, + "per_stream_tps": 268.6, + "timing_valid": true, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 268.88819861334224, + "per_stream_tps": [ + 268.9 + ], + "makespan_s": 0.9719928340055048, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c67af3b924400b5" + ], + "comparison_refs": [ + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + } + ] + }, + { + "aggregate_tps": 268.2399437943784, + "per_stream_tps": [ + 268.2 + ], + "makespan_s": 0.9742348329164088, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c67af3b924400b5" + ], + "comparison_refs": [ + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + } + ] + }, + { + "aggregate_tps": 268.6443575396984, + "per_stream_tps": [ + 268.6 + ], + "makespan_s": 0.9728733748197556, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c67af3b924400b5" + ], + "comparison_refs": [ + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 788.5, + "per_stream_tps": 202.5, + "timing_valid": true, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 787.4625669810956, + "per_stream_tps": [ + 204.2, + 196.9, + 200.5, + 208.0 + ], + "makespan_s": 1.3196133328601718, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c67af3b924400b5", + "0c67af3b924400b5", + "0c67af3b924400b5", + "0c67af3b924400b5" + ], + "comparison_refs": [ + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + }, + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + }, + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + }, + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + } + ] + }, + { + "aggregate_tps": 788.5080115844421, + "per_stream_tps": [ + 208.3, + 197.1, + 204.4, + 200.7 + ], + "makespan_s": 1.318005375098437, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c67af3b924400b5", + "0c67af3b924400b5", + "0c67af3b924400b5", + "0c67af3b924400b5" + ], + "comparison_refs": [ + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + }, + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + }, + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + }, + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + } + ] + }, + { + "aggregate_tps": 788.510017526478, + "per_stream_tps": [ + 200.8, + 197.2, + 208.3, + 204.5 + ], + "makespan_s": 1.3175547090359032, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c67af3b924400b5", + "0c67af3b924400b5", + "0c67af3b924400b5", + "0c67af3b924400b5" + ], + "comparison_refs": [ + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + }, + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + }, + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + }, + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + } + ] + } + ] + } + ] + }, + { + "family": "vl", + "bits": 6, + "leg": 1, + "engine": "standard", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.16758570913225412, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "5be03ca3c8afb2ef00d7a9bca70175977199013f91113b4a9cd3750353f25182", + "token_sha256": null + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.39433749997988343, + "output_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "request_body_sha256": "d38d5597bc883e9609612a3617f30a9a667bfa90402aacf0db9271de1a89adc8", + "token_sha256": null + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "unverified if server omits token hashes; text digest is not substituted" + }, + "memory": { + "max_rss_bytes": 2292514816, + "peak_physical_footprint_bytes": 3587885792 + }, + "tag": "vl-6-1-standard", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 286.0, + "per_stream_tps": 286.0, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 286.66413162968576, + "per_stream_tps": [ + 286.7 + ], + "makespan_s": 0.9085515001788735, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "47e27a882f6b6bc0" + ], + "comparison_refs": [ + { + "concurrent": "44111736ac335976", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + } + ] + }, + { + "aggregate_tps": 286.00340004924624, + "per_stream_tps": [ + 286.0 + ], + "makespan_s": 0.9105483330786228, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "47e27a882f6b6bc0" + ], + "comparison_refs": [ + { + "concurrent": "44111736ac335976", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + } + ] + }, + { + "aggregate_tps": 280.0785225529897, + "per_stream_tps": [ + 280.1 + ], + "makespan_s": 0.9297830406576395, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "47e27a882f6b6bc0" + ], + "comparison_refs": [ + { + "concurrent": "44111736ac335976", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 920.7, + "per_stream_tps": 233.8, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 916.0021880677776, + "per_stream_tps": [ + 230.9, + 233.4, + 233.3, + 233.3 + ], + "makespan_s": 1.1514438749291003, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a5a6c16baa858220", + "53ab6a9ba0132b24", + "53ab6a9ba0132b24", + "53ab6a9ba0132b24" + ], + "comparison_refs": [ + { + "concurrent": "4ff40deab3b3ee42", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + }, + { + "concurrent": "d77d5b2340f0e3d8", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + }, + { + "concurrent": "d77d5b2340f0e3d8", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + }, + { + "concurrent": "d77d5b2340f0e3d8", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + } + ] + }, + { + "aggregate_tps": 921.1190932036707, + "per_stream_tps": [ + 234.3, + 234.3, + 232.1, + 234.3 + ], + "makespan_s": 1.1419340413995087, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "53ab6a9ba0132b24", + "53ab6a9ba0132b24", + "a5a6c16baa858220", + "53ab6a9ba0132b24" + ], + "comparison_refs": [ + { + "concurrent": "d77d5b2340f0e3d8", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + }, + { + "concurrent": "d77d5b2340f0e3d8", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + }, + { + "concurrent": "4ff40deab3b3ee42", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + }, + { + "concurrent": "d77d5b2340f0e3d8", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + } + ] + }, + { + "aggregate_tps": 920.6609174889724, + "per_stream_tps": [ + 234.2, + 232.0, + 234.2, + 234.2 + ], + "makespan_s": 1.1422320413403213, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "53ab6a9ba0132b24", + "a5a6c16baa858220", + "53ab6a9ba0132b24", + "53ab6a9ba0132b24" + ], + "comparison_refs": [ + { + "concurrent": "d77d5b2340f0e3d8", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + }, + { + "concurrent": "4ff40deab3b3ee42", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + }, + { + "concurrent": "d77d5b2340f0e3d8", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + }, + { + "concurrent": "d77d5b2340f0e3d8", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 280.9, + "per_stream_tps": 280.9, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 280.85344950568253, + "per_stream_tps": [ + 280.9 + ], + "makespan_s": 0.9269298748113215, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "78d0d48406b30e24" + ], + "comparison_refs": [ + { + "concurrent": "f7c90614f10021f0", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + } + ] + }, + { + "aggregate_tps": 280.66937810380546, + "per_stream_tps": [ + 280.7 + ], + "makespan_s": 0.9292557500302792, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "78d0d48406b30e24" + ], + "comparison_refs": [ + { + "concurrent": "f7c90614f10021f0", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + } + ] + }, + { + "aggregate_tps": 281.8581193887578, + "per_stream_tps": [ + 281.9 + ], + "makespan_s": 0.9253666247241199, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "78d0d48406b30e24" + ], + "comparison_refs": [ + { + "concurrent": "f7c90614f10021f0", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 921.8, + "per_stream_tps": 234.3, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 919.8637136308564, + "per_stream_tps": [ + 231.8, + 234.1, + 234.1, + 234.1 + ], + "makespan_s": 1.145006874576211, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a36c6a6fd4f53222", + "79c7bb374559ff06", + "79c7bb374559ff06", + "79c7bb374559ff06" + ], + "comparison_refs": [ + { + "concurrent": "8bab9469ba3b5430", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + }, + { + "concurrent": "87e8945794a1bed6", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + }, + { + "concurrent": "87e8945794a1bed6", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + }, + { + "concurrent": "87e8945794a1bed6", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + } + ] + }, + { + "aggregate_tps": 921.7825690900612, + "per_stream_tps": [ + 234.5, + 234.5, + 234.5, + 232.3 + ], + "makespan_s": 1.141130750067532, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "79c7bb374559ff06", + "79c7bb374559ff06", + "79c7bb374559ff06", + "a36c6a6fd4f53222" + ], + "comparison_refs": [ + { + "concurrent": "87e8945794a1bed6", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + }, + { + "concurrent": "87e8945794a1bed6", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + }, + { + "concurrent": "87e8945794a1bed6", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + }, + { + "concurrent": "8bab9469ba3b5430", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + } + ] + }, + { + "aggregate_tps": 922.3224407904422, + "per_stream_tps": [ + 234.6, + 234.6, + 232.4, + 234.6 + ], + "makespan_s": 1.14044183306396, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "79c7bb374559ff06", + "79c7bb374559ff06", + "a36c6a6fd4f53222", + "79c7bb374559ff06" + ], + "comparison_refs": [ + { + "concurrent": "87e8945794a1bed6", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + }, + { + "concurrent": "87e8945794a1bed6", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + }, + { + "concurrent": "8bab9469ba3b5430", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + }, + { + "concurrent": "87e8945794a1bed6", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + } + ] + } + ] + } + ] + }, + { + "family": "vl", + "bits": 6, + "leg": 2, + "engine": "standard", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.1679111672565341, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "5be03ca3c8afb2ef00d7a9bca70175977199013f91113b4a9cd3750353f25182", + "token_sha256": null + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.39429708290845156, + "output_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "request_body_sha256": "d38d5597bc883e9609612a3617f30a9a667bfa90402aacf0db9271de1a89adc8", + "token_sha256": null + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "unverified if server omits token hashes; text digest is not substituted" + }, + "memory": { + "max_rss_bytes": 2292301824, + "peak_physical_footprint_bytes": 3590556504 + }, + "tag": "vl-6-2-standard", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 282.0, + "per_stream_tps": 282.0, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 282.2657284112393, + "per_stream_tps": [ + 282.3 + ], + "makespan_s": 0.9237472089007497, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "47e27a882f6b6bc0" + ], + "comparison_refs": [ + { + "concurrent": "44111736ac335976", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + } + ] + }, + { + "aggregate_tps": 281.95779715464187, + "per_stream_tps": [ + 282.0 + ], + "makespan_s": 0.9249823749996722, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "47e27a882f6b6bc0" + ], + "comparison_refs": [ + { + "concurrent": "44111736ac335976", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + } + ] + }, + { + "aggregate_tps": 281.1160564368018, + "per_stream_tps": [ + 281.1 + ], + "makespan_s": 0.9276661658659577, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "47e27a882f6b6bc0" + ], + "comparison_refs": [ + { + "concurrent": "44111736ac335976", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 920.3, + "per_stream_tps": 233.9, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 917.3428733128613, + "per_stream_tps": [ + 233.7, + 233.7, + 233.7, + 231.2 + ], + "makespan_s": 1.1505417078733444, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "53ab6a9ba0132b24", + "53ab6a9ba0132b24", + "53ab6a9ba0132b24", + "a5a6c16baa858220" + ], + "comparison_refs": [ + { + "concurrent": "d77d5b2340f0e3d8", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + }, + { + "concurrent": "d77d5b2340f0e3d8", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + }, + { + "concurrent": "d77d5b2340f0e3d8", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + }, + { + "concurrent": "4ff40deab3b3ee42", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + } + ] + }, + { + "aggregate_tps": 920.5580579793124, + "per_stream_tps": [ + 231.9, + 234.2, + 234.2, + 234.2 + ], + "makespan_s": 1.1429185410961509, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a5a6c16baa858220", + "53ab6a9ba0132b24", + "53ab6a9ba0132b24", + "53ab6a9ba0132b24" + ], + "comparison_refs": [ + { + "concurrent": "4ff40deab3b3ee42", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + }, + { + "concurrent": "d77d5b2340f0e3d8", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + }, + { + "concurrent": "d77d5b2340f0e3d8", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + }, + { + "concurrent": "d77d5b2340f0e3d8", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + } + ] + }, + { + "aggregate_tps": 920.2576103010215, + "per_stream_tps": [ + 231.9, + 234.1, + 234.1, + 234.1 + ], + "makespan_s": 1.1423820410855114, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a5a6c16baa858220", + "53ab6a9ba0132b24", + "53ab6a9ba0132b24", + "53ab6a9ba0132b24" + ], + "comparison_refs": [ + { + "concurrent": "4ff40deab3b3ee42", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + }, + { + "concurrent": "d77d5b2340f0e3d8", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + }, + { + "concurrent": "d77d5b2340f0e3d8", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + }, + { + "concurrent": "d77d5b2340f0e3d8", + "alone": "44111736ac335976", + "serial": "44111736ac335976" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 281.9, + "per_stream_tps": 281.9, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 281.93855988587035, + "per_stream_tps": [ + 281.9 + ], + "makespan_s": 0.9246124997735023, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "78d0d48406b30e24" + ], + "comparison_refs": [ + { + "concurrent": "f7c90614f10021f0", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + } + ] + }, + { + "aggregate_tps": 281.23047151973066, + "per_stream_tps": [ + 281.2 + ], + "makespan_s": 0.9273595409467816, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "78d0d48406b30e24" + ], + "comparison_refs": [ + { + "concurrent": "f7c90614f10021f0", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + } + ] + }, + { + "aggregate_tps": 282.01473172593103, + "per_stream_tps": [ + 282.0 + ], + "makespan_s": 0.9244530834257603, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "78d0d48406b30e24" + ], + "comparison_refs": [ + { + "concurrent": "f7c90614f10021f0", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 922.8, + "per_stream_tps": 234.3, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 918.4030066473836, + "per_stream_tps": [ + 233.8, + 233.8, + 231.4, + 233.8 + ], + "makespan_s": 1.14735791599378, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "79c7bb374559ff06", + "79c7bb374559ff06", + "a36c6a6fd4f53222", + "79c7bb374559ff06" + ], + "comparison_refs": [ + { + "concurrent": "87e8945794a1bed6", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + }, + { + "concurrent": "87e8945794a1bed6", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + }, + { + "concurrent": "8bab9469ba3b5430", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + }, + { + "concurrent": "87e8945794a1bed6", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + } + ] + }, + { + "aggregate_tps": 922.776637284845, + "per_stream_tps": [ + 234.8, + 234.8, + 232.5, + 234.8 + ], + "makespan_s": 1.139778083190322, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "79c7bb374559ff06", + "79c7bb374559ff06", + "a36c6a6fd4f53222", + "79c7bb374559ff06" + ], + "comparison_refs": [ + { + "concurrent": "87e8945794a1bed6", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + }, + { + "concurrent": "87e8945794a1bed6", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + }, + { + "concurrent": "8bab9469ba3b5430", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + }, + { + "concurrent": "87e8945794a1bed6", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + } + ] + }, + { + "aggregate_tps": 923.9603297120502, + "per_stream_tps": [ + 232.9, + 235.0, + 235.0, + 235.0 + ], + "makespan_s": 1.1382407080382109, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a36c6a6fd4f53222", + "79c7bb374559ff06", + "79c7bb374559ff06", + "79c7bb374559ff06" + ], + "comparison_refs": [ + { + "concurrent": "8bab9469ba3b5430", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + }, + { + "concurrent": "87e8945794a1bed6", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + }, + { + "concurrent": "87e8945794a1bed6", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + }, + { + "concurrent": "87e8945794a1bed6", + "alone": "f7c90614f10021f0", + "serial": "f7c90614f10021f0" + } + ] + } + ] + } + ] + }, + { + "family": "vl", + "bits": 6, + "leg": 3, + "engine": "current", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.9580447501502931, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27", + "token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.986285790801048, + "output_sha256": "818f0110cd9290761c90436e4568f7b0e21b1f87e47edaca794399487762711e", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0", + "token_sha256": "a57cf1b311a1c093ba93fafd5f45702e8debf3b823b0bd975adbf64a79038cc0" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 2026733568, + "peak_physical_footprint_bytes": 2483505480 + }, + "tag": "vl-6-3-current", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 271.5, + "per_stream_tps": 271.5, + "timing_valid": true, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 271.53563774123603, + "per_stream_tps": [ + 271.5 + ], + "makespan_s": 0.9625334171578288, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "ff5c2a57d0d6ba68" + ], + "comparison_refs": [ + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + } + ] + }, + { + "aggregate_tps": 271.08271625888136, + "per_stream_tps": [ + 271.1 + ], + "makespan_s": 0.9640820831991732, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "ff5c2a57d0d6ba68" + ], + "comparison_refs": [ + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + } + ] + }, + { + "aggregate_tps": 272.0056939363307, + "per_stream_tps": [ + 272.0 + ], + "makespan_s": 0.960909207817167, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "ff5c2a57d0d6ba68" + ], + "comparison_refs": [ + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 794.4, + "per_stream_tps": 204.2, + "timing_valid": true, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 794.1989479492437, + "per_stream_tps": [ + 202.2, + 205.9, + 209.8, + 198.6 + ], + "makespan_s": 1.3084426247514784, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "ff5c2a57d0d6ba68", + "ff5c2a57d0d6ba68", + "ff5c2a57d0d6ba68", + "ff5c2a57d0d6ba68" + ], + "comparison_refs": [ + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + }, + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + }, + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + }, + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + } + ] + }, + { + "aggregate_tps": 794.8943623671728, + "per_stream_tps": [ + 198.7, + 202.4, + 210.0, + 206.1 + ], + "makespan_s": 1.3072670004330575, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "ff5c2a57d0d6ba68", + "ff5c2a57d0d6ba68", + "ff5c2a57d0d6ba68", + "ff5c2a57d0d6ba68" + ], + "comparison_refs": [ + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + }, + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + }, + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + }, + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + } + ] + }, + { + "aggregate_tps": 794.4005403579406, + "per_stream_tps": [ + 198.7, + 209.8, + 202.3, + 206.0 + ], + "makespan_s": 1.3079056669957936, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "ff5c2a57d0d6ba68", + "ff5c2a57d0d6ba68", + "ff5c2a57d0d6ba68", + "ff5c2a57d0d6ba68" + ], + "comparison_refs": [ + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + }, + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + }, + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + }, + { + "concurrent": "d22d0b8c96f314fb", + "alone": "d22d0b8c96f314fb", + "serial": "d22d0b8c96f314fb" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 267.7, + "per_stream_tps": 267.7, + "timing_valid": true, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 267.47826222430797, + "per_stream_tps": [ + 267.5 + ], + "makespan_s": 0.9770845002494752, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c67af3b924400b5" + ], + "comparison_refs": [ + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + } + ] + }, + { + "aggregate_tps": 267.69837267617953, + "per_stream_tps": [ + 267.7 + ], + "makespan_s": 0.9761787499301136, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c67af3b924400b5" + ], + "comparison_refs": [ + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + } + ] + }, + { + "aggregate_tps": 267.8347645926309, + "per_stream_tps": [ + 267.8 + ], + "makespan_s": 0.9757044170983136, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c67af3b924400b5" + ], + "comparison_refs": [ + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 788.5, + "per_stream_tps": 202.6, + "timing_valid": true, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 787.4317146856454, + "per_stream_tps": [ + 208.0, + 204.2, + 200.5, + 196.9 + ], + "makespan_s": 1.3193254582583904, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c67af3b924400b5", + "0c67af3b924400b5", + "0c67af3b924400b5", + "0c67af3b924400b5" + ], + "comparison_refs": [ + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + }, + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + }, + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + }, + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + } + ] + }, + { + "aggregate_tps": 788.4807599415163, + "per_stream_tps": [ + 204.4, + 208.3, + 197.2, + 200.8 + ], + "makespan_s": 1.3175824168138206, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c67af3b924400b5", + "0c67af3b924400b5", + "0c67af3b924400b5", + "0c67af3b924400b5" + ], + "comparison_refs": [ + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + }, + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + }, + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + }, + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + } + ] + }, + { + "aggregate_tps": 789.1557876569545, + "per_stream_tps": [ + 208.5, + 204.6, + 200.9, + 197.3 + ], + "makespan_s": 1.3168232501484454, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "0c67af3b924400b5", + "0c67af3b924400b5", + "0c67af3b924400b5", + "0c67af3b924400b5" + ], + "comparison_refs": [ + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + }, + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + }, + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + }, + { + "concurrent": "7fedd0adca47803e", + "alone": "7fedd0adca47803e", + "serial": "7fedd0adca47803e" + } + ] + } + ] + } + ] + }, + { + "family": "vl", + "bits": 8, + "leg": 0, + "engine": "current", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.848729083314538, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27", + "token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.514424624852836, + "output_sha256": "b7b834e8f232d6ff122c84f11c241563a5b5bd6ae97dfaabd0e38994edaff19e", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0", + "token_sha256": "8ed4e05be9f053ca054d2c47326a04f0d7fb0c230374bc8bbafaa9618d59ed38" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 2458845184, + "peak_physical_footprint_bytes": 2915682920 + }, + "tag": "vl-8-0-current", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 261.4, + "per_stream_tps": 261.4, + "timing_valid": true, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 261.43023610999523, + "per_stream_tps": [ + 261.4 + ], + "makespan_s": 0.9962034579366446, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "aa724824fb24493a" + ], + "comparison_refs": [ + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + } + ] + }, + { + "aggregate_tps": 261.40561385841306, + "per_stream_tps": [ + 261.4 + ], + "makespan_s": 0.9962740419432521, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "aa724824fb24493a" + ], + "comparison_refs": [ + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + } + ] + }, + { + "aggregate_tps": 261.8779627488741, + "per_stream_tps": [ + 261.9 + ], + "makespan_s": 0.9944937918335199, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "aa724824fb24493a" + ], + "comparison_refs": [ + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 805.7, + "per_stream_tps": 206.4, + "timing_valid": true, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 805.4979180701675, + "per_stream_tps": [ + 208.0, + 201.4, + 204.7, + 211.5 + ], + "makespan_s": 1.2877319580875337, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "aa724824fb24493a", + "aa724824fb24493a", + "aa724824fb24493a", + "aa724824fb24493a" + ], + "comparison_refs": [ + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + }, + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + }, + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + }, + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + } + ] + }, + { + "aggregate_tps": 805.7061110891989, + "per_stream_tps": [ + 204.8, + 211.6, + 201.5, + 208.2 + ], + "makespan_s": 1.2870489582419395, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "aa724824fb24493a", + "aa724824fb24493a", + "aa724824fb24493a", + "aa724824fb24493a" + ], + "comparison_refs": [ + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + }, + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + }, + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + }, + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + } + ] + }, + { + "aggregate_tps": 806.288683138432, + "per_stream_tps": [ + 201.6, + 208.2, + 211.7, + 204.9 + ], + "makespan_s": 1.286464957986027, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "aa724824fb24493a", + "aa724824fb24493a", + "aa724824fb24493a", + "aa724824fb24493a" + ], + "comparison_refs": [ + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + }, + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + }, + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + }, + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 261.5, + "per_stream_tps": 261.5, + "timing_valid": true, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 261.257944162626, + "per_stream_tps": [ + 261.3 + ], + "makespan_s": 0.9969625407829881, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a957c3a797c87e88" + ], + "comparison_refs": [ + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + } + ] + }, + { + "aggregate_tps": 261.51485806633576, + "per_stream_tps": [ + 261.5 + ], + "makespan_s": 0.9960329169407487, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a957c3a797c87e88" + ], + "comparison_refs": [ + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + } + ] + }, + { + "aggregate_tps": 261.63581473828185, + "per_stream_tps": [ + 261.6 + ], + "makespan_s": 0.995581166818738, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a957c3a797c87e88" + ], + "comparison_refs": [ + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 805.9, + "per_stream_tps": 206.6, + "timing_valid": true, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 805.5417855137309, + "per_stream_tps": [ + 204.7, + 208.1, + 211.7, + 201.4 + ], + "makespan_s": 1.2879021661356091, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a957c3a797c87e88", + "a957c3a797c87e88", + "a957c3a797c87e88", + "a957c3a797c87e88" + ], + "comparison_refs": [ + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + }, + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + }, + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + }, + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + } + ] + }, + { + "aggregate_tps": 805.9301706929242, + "per_stream_tps": [ + 204.9, + 211.8, + 208.3, + 201.6 + ], + "makespan_s": 1.2867747908458114, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a957c3a797c87e88", + "a957c3a797c87e88", + "a957c3a797c87e88", + "a957c3a797c87e88" + ], + "comparison_refs": [ + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + }, + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + }, + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + }, + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + } + ] + }, + { + "aggregate_tps": 807.124003540818, + "per_stream_tps": [ + 205.2, + 208.5, + 212.1, + 201.8 + ], + "makespan_s": 1.2850457918830216, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a957c3a797c87e88", + "a957c3a797c87e88", + "a957c3a797c87e88", + "a957c3a797c87e88" + ], + "comparison_refs": [ + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + }, + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + }, + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + }, + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + } + ] + } + ] + } + ] + }, + { + "family": "vl", + "bits": 8, + "leg": 1, + "engine": "standard", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.16829495783895254, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "c2a4c1cc3596cec1d1582d1c924eb343ab9a3d1c9ba1cb02072c3194336118d7", + "token_sha256": null + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.38476558309048414, + "output_sha256": "b7b834e8f232d6ff122c84f11c241563a5b5bd6ae97dfaabd0e38994edaff19e", + "request_body_sha256": "6c4d58b058deeae1ab4f4ad705c0142e313b2e7d5b5cba35879fb784dc274280", + "token_sha256": null + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "unverified if server omits token hashes; text digest is not substituted" + }, + "memory": { + "max_rss_bytes": 2724462592, + "peak_physical_footprint_bytes": 4021373952 + }, + "tag": "vl-8-1-standard", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 272.6, + "per_stream_tps": 272.6, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 271.9296002017262, + "per_stream_tps": [ + 271.9 + ], + "makespan_s": 0.9568977919407189, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "15b30bd9b33e5220" + ], + "comparison_refs": [ + { + "concurrent": "ffff87b6ee89dd36", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + } + ] + }, + { + "aggregate_tps": 272.6069684100229, + "per_stream_tps": [ + 272.6 + ], + "makespan_s": 0.9548117499798536, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "15b30bd9b33e5220" + ], + "comparison_refs": [ + { + "concurrent": "ffff87b6ee89dd36", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + } + ] + }, + { + "aggregate_tps": 272.6057175997051, + "per_stream_tps": [ + 272.6 + ], + "makespan_s": 0.9547965000383556, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "15b30bd9b33e5220" + ], + "comparison_refs": [ + { + "concurrent": "ffff87b6ee89dd36", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 921.6, + "per_stream_tps": 233.4, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 911.8759775283856, + "per_stream_tps": [ + 229.8, + 232.3, + 232.3, + 232.3 + ], + "makespan_s": 1.1538290833123028, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "1c2c166eae95d7e6", + "27e6e32ca698e6d7", + "27e6e32ca698e6d7", + "27e6e32ca698e6d7" + ], + "comparison_refs": [ + { + "concurrent": "4be982359420377d", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + }, + { + "concurrent": "b1473411bdc7c6a4", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + }, + { + "concurrent": "b1473411bdc7c6a4", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + }, + { + "concurrent": "b1473411bdc7c6a4", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + } + ] + }, + { + "aggregate_tps": 921.6035392915104, + "per_stream_tps": [ + 234.5, + 234.5, + 232.3, + 234.5 + ], + "makespan_s": 1.1414858330972493, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "27e6e32ca698e6d7", + "27e6e32ca698e6d7", + "1c2c166eae95d7e6", + "27e6e32ca698e6d7" + ], + "comparison_refs": [ + { + "concurrent": "b1473411bdc7c6a4", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + }, + { + "concurrent": "b1473411bdc7c6a4", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + }, + { + "concurrent": "4be982359420377d", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + }, + { + "concurrent": "b1473411bdc7c6a4", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + } + ] + }, + { + "aggregate_tps": 921.7025718898061, + "per_stream_tps": [ + 232.3, + 234.5, + 234.5, + 234.5 + ], + "makespan_s": 1.1410312922671437, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "1c2c166eae95d7e6", + "27e6e32ca698e6d7", + "27e6e32ca698e6d7", + "27e6e32ca698e6d7" + ], + "comparison_refs": [ + { + "concurrent": "4be982359420377d", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + }, + { + "concurrent": "b1473411bdc7c6a4", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + }, + { + "concurrent": "b1473411bdc7c6a4", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + }, + { + "concurrent": "b1473411bdc7c6a4", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 266.7, + "per_stream_tps": 266.7, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 266.1758939251264, + "per_stream_tps": [ + 266.2 + ], + "makespan_s": 0.9791338327340782, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "6388028581e2b97b" + ], + "comparison_refs": [ + { + "concurrent": "75a9f00ebfc68dba", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + } + ] + }, + { + "aggregate_tps": 266.71546561982575, + "per_stream_tps": [ + 266.7 + ], + "makespan_s": 0.9777331668883562, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "6388028581e2b97b" + ], + "comparison_refs": [ + { + "concurrent": "75a9f00ebfc68dba", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + } + ] + }, + { + "aggregate_tps": 266.72389322258886, + "per_stream_tps": [ + 266.7 + ], + "makespan_s": 0.9770145416259766, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "6388028581e2b97b" + ], + "comparison_refs": [ + { + "concurrent": "75a9f00ebfc68dba", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 919.4, + "per_stream_tps": 233.7, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 916.7136443009506, + "per_stream_tps": [ + 233.4, + 233.4, + 233.4, + 231.0 + ], + "makespan_s": 1.1500630001537502, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "02d9b2248a4ea3f5", + "02d9b2248a4ea3f5", + "02d9b2248a4ea3f5", + "01c067355e9065bb" + ], + "comparison_refs": [ + { + "concurrent": "8d184bece38566a4", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + }, + { + "concurrent": "8d184bece38566a4", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + }, + { + "concurrent": "8d184bece38566a4", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + }, + { + "concurrent": "c64a0f957d03e6e7", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + } + ] + }, + { + "aggregate_tps": 919.422737479508, + "per_stream_tps": [ + 233.9, + 233.9, + 231.7, + 233.9 + ], + "makespan_s": 1.1439807917922735, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "02d9b2248a4ea3f5", + "02d9b2248a4ea3f5", + "01c067355e9065bb", + "02d9b2248a4ea3f5" + ], + "comparison_refs": [ + { + "concurrent": "8d184bece38566a4", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + }, + { + "concurrent": "8d184bece38566a4", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + }, + { + "concurrent": "c64a0f957d03e6e7", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + }, + { + "concurrent": "8d184bece38566a4", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + } + ] + }, + { + "aggregate_tps": 919.6839409277534, + "per_stream_tps": [ + 234.0, + 234.0, + 231.8, + 234.0 + ], + "makespan_s": 1.1434024590998888, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "02d9b2248a4ea3f5", + "02d9b2248a4ea3f5", + "01c067355e9065bb", + "02d9b2248a4ea3f5" + ], + "comparison_refs": [ + { + "concurrent": "8d184bece38566a4", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + }, + { + "concurrent": "8d184bece38566a4", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + }, + { + "concurrent": "c64a0f957d03e6e7", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + }, + { + "concurrent": "8d184bece38566a4", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + } + ] + } + ] + } + ] + }, + { + "family": "vl", + "bits": 8, + "leg": 2, + "engine": "standard", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.16713600000366569, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "c2a4c1cc3596cec1d1582d1c924eb343ab9a3d1c9ba1cb02072c3194336118d7", + "token_sha256": null + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.3835990000516176, + "output_sha256": "b7b834e8f232d6ff122c84f11c241563a5b5bd6ae97dfaabd0e38994edaff19e", + "request_body_sha256": "6c4d58b058deeae1ab4f4ad705c0142e313b2e7d5b5cba35879fb784dc274280", + "token_sha256": null + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "unverified if server omits token hashes; text digest is not substituted" + }, + "memory": { + "max_rss_bytes": 2724315136, + "peak_physical_footprint_bytes": 4021898192 + }, + "tag": "vl-8-2-standard", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 272.8, + "per_stream_tps": 272.8, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 272.78536728501274, + "per_stream_tps": [ + 272.8 + ], + "makespan_s": 0.9541327077895403, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "15b30bd9b33e5220" + ], + "comparison_refs": [ + { + "concurrent": "ffff87b6ee89dd36", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + } + ] + }, + { + "aggregate_tps": 272.73241373565884, + "per_stream_tps": [ + 272.7 + ], + "makespan_s": 0.9542159996926785, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "15b30bd9b33e5220" + ], + "comparison_refs": [ + { + "concurrent": "ffff87b6ee89dd36", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + } + ] + }, + { + "aggregate_tps": 272.9132160463104, + "per_stream_tps": [ + 272.9 + ], + "makespan_s": 0.9536322499625385, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "15b30bd9b33e5220" + ], + "comparison_refs": [ + { + "concurrent": "ffff87b6ee89dd36", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 920.4, + "per_stream_tps": 233.1, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 910.164902771632, + "per_stream_tps": [ + 229.4, + 231.8, + 231.8, + 231.8 + ], + "makespan_s": 1.1562817497178912, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "1c2c166eae95d7e6", + "27e6e32ca698e6d7", + "27e6e32ca698e6d7", + "27e6e32ca698e6d7" + ], + "comparison_refs": [ + { + "concurrent": "4be982359420377d", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + }, + { + "concurrent": "b1473411bdc7c6a4", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + }, + { + "concurrent": "b1473411bdc7c6a4", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + }, + { + "concurrent": "b1473411bdc7c6a4", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + } + ] + }, + { + "aggregate_tps": 921.5829305350238, + "per_stream_tps": [ + 232.2, + 234.5, + 234.5, + 234.5 + ], + "makespan_s": 1.141255334019661, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "1c2c166eae95d7e6", + "27e6e32ca698e6d7", + "27e6e32ca698e6d7", + "27e6e32ca698e6d7" + ], + "comparison_refs": [ + { + "concurrent": "4be982359420377d", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + }, + { + "concurrent": "b1473411bdc7c6a4", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + }, + { + "concurrent": "b1473411bdc7c6a4", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + }, + { + "concurrent": "b1473411bdc7c6a4", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + } + ] + }, + { + "aggregate_tps": 920.3645196131935, + "per_stream_tps": [ + 234.1, + 234.1, + 234.1, + 232.0 + ], + "makespan_s": 1.1428453330881894, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "27e6e32ca698e6d7", + "27e6e32ca698e6d7", + "27e6e32ca698e6d7", + "1c2c166eae95d7e6" + ], + "comparison_refs": [ + { + "concurrent": "b1473411bdc7c6a4", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + }, + { + "concurrent": "b1473411bdc7c6a4", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + }, + { + "concurrent": "b1473411bdc7c6a4", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + }, + { + "concurrent": "4be982359420377d", + "alone": "ffff87b6ee89dd36", + "serial": "ffff87b6ee89dd36" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 267.0, + "per_stream_tps": 267.0, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 3 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 266.96923179557734, + "per_stream_tps": [ + 267.0 + ], + "makespan_s": 0.9762948751449585, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "6388028581e2b97b" + ], + "comparison_refs": [ + { + "concurrent": "75a9f00ebfc68dba", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + } + ] + }, + { + "aggregate_tps": 267.2788339669485, + "per_stream_tps": [ + 267.3 + ], + "makespan_s": 0.975237709004432, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "6388028581e2b97b" + ], + "comparison_refs": [ + { + "concurrent": "75a9f00ebfc68dba", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + } + ] + }, + { + "aggregate_tps": 266.98353384406676, + "per_stream_tps": [ + 267.0 + ], + "makespan_s": 0.9763321247883141, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "6388028581e2b97b" + ], + "comparison_refs": [ + { + "concurrent": "75a9f00ebfc68dba", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 920.0, + "per_stream_tps": 233.4, + "timing_valid": true, + "alone": { + "equal": 0, + "unequal": 0, + "failed": 12 + }, + "serial": { + "equal": 0, + "unequal": 0, + "failed": 1 + }, + "reps": [ + { + "aggregate_tps": 914.0742569393119, + "per_stream_tps": [ + 232.8, + 232.8, + 232.8, + 230.4 + ], + "makespan_s": 1.1548401252366602, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "02d9b2248a4ea3f5", + "02d9b2248a4ea3f5", + "02d9b2248a4ea3f5", + "01c067355e9065bb" + ], + "comparison_refs": [ + { + "concurrent": "8d184bece38566a4", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + }, + { + "concurrent": "8d184bece38566a4", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + }, + { + "concurrent": "8d184bece38566a4", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + }, + { + "concurrent": "c64a0f957d03e6e7", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + } + ] + }, + { + "aggregate_tps": 919.9558035326796, + "per_stream_tps": [ + 234.1, + 234.1, + 234.1, + 231.9 + ], + "makespan_s": 1.1439747922122478, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "02d9b2248a4ea3f5", + "02d9b2248a4ea3f5", + "02d9b2248a4ea3f5", + "01c067355e9065bb" + ], + "comparison_refs": [ + { + "concurrent": "8d184bece38566a4", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + }, + { + "concurrent": "8d184bece38566a4", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + }, + { + "concurrent": "8d184bece38566a4", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + }, + { + "concurrent": "c64a0f957d03e6e7", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + } + ] + }, + { + "aggregate_tps": 920.3013062992629, + "per_stream_tps": [ + 234.1, + 234.2, + 231.9, + 234.2 + ], + "makespan_s": 1.1431216252967715, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "02d9b2248a4ea3f5", + "02d9b2248a4ea3f5", + "01c067355e9065bb", + "02d9b2248a4ea3f5" + ], + "comparison_refs": [ + { + "concurrent": "8d184bece38566a4", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + }, + { + "concurrent": "8d184bece38566a4", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + }, + { + "concurrent": "c64a0f957d03e6e7", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + }, + { + "concurrent": "8d184bece38566a4", + "alone": "75a9f00ebfc68dba", + "serial": "75a9f00ebfc68dba" + } + ] + } + ] + } + ] + }, + { + "family": "vl", + "bits": 8, + "leg": 3, + "engine": "current", + "cold": [ + { + "prompt_tokens": 2048, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 0.8491029171273112, + "output_sha256": "601e2aa660719b848a656983445b3d821acf6a7c7eafca375f4fbac08a72298e", + "request_body_sha256": "bb5a87e435a323373c42953ab2e62d1b483f9f30c502cf8102ead10409525b27", + "token_sha256": "21c3879cda05ccf4cde74c1f54d326d2c271f673656d166241e6643c21f49d89" + }, + { + "prompt_tokens": 8192, + "completion_tokens": 16, + "cached_tokens": 0, + "ttft_s": 4.512903375085443, + "output_sha256": "b7b834e8f232d6ff122c84f11c241563a5b5bd6ae97dfaabd0e38994edaff19e", + "request_body_sha256": "1df0a7343db5cb48b65f713a1410e84f5b48748d7e09bf32686bacc3e4bbb5e0", + "token_sha256": "8ed4e05be9f053ca054d2c47326a04f0d7fb0c230374bc8bbafaa9618d59ed38" + } + ], + "decode_validation": { + "measured_requests": 30, + "all_256": true, + "standard_token_hash_equality": "validated by existing client" + }, + "memory": { + "max_rss_bytes": 2458812416, + "peak_physical_footprint_bytes": 2915699280 + }, + "tag": "vl-8-3-current", + "decode_cells": [ + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 261.5, + "per_stream_tps": 261.5, + "timing_valid": true, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 261.7591559238837, + "per_stream_tps": [ + 261.8 + ], + "makespan_s": 0.9948987080715597, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "aa724824fb24493a" + ], + "comparison_refs": [ + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + } + ] + }, + { + "aggregate_tps": 261.40418483699045, + "per_stream_tps": [ + 261.4 + ], + "makespan_s": 0.9962387080304325, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "aa724824fb24493a" + ], + "comparison_refs": [ + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + } + ] + }, + { + "aggregate_tps": 261.5214178780997, + "per_stream_tps": [ + 261.5 + ], + "makespan_s": 0.9958214578218758, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "aa724824fb24493a" + ], + "comparison_refs": [ + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + } + ] + } + ] + }, + { + "prompt": "long-code-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 808.1, + "per_stream_tps": 206.8, + "timing_valid": true, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 806.1159173191214, + "per_stream_tps": [ + 201.6, + 204.9, + 211.7, + 208.2 + ], + "makespan_s": 1.2864587092772126, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "aa724824fb24493a", + "aa724824fb24493a", + "aa724824fb24493a", + "aa724824fb24493a" + ], + "comparison_refs": [ + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + }, + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + }, + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + }, + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + } + ] + }, + { + "aggregate_tps": 808.0743795504492, + "per_stream_tps": [ + 205.4, + 212.3, + 208.7, + 202.1 + ], + "makespan_s": 1.2836079159751534, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "aa724824fb24493a", + "aa724824fb24493a", + "aa724824fb24493a", + "aa724824fb24493a" + ], + "comparison_refs": [ + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + }, + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + }, + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + }, + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + } + ] + }, + { + "aggregate_tps": 808.1363220557455, + "per_stream_tps": [ + 202.1, + 212.2, + 205.4, + 208.8 + ], + "makespan_s": 1.2834009169600904, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "aa724824fb24493a", + "aa724824fb24493a", + "aa724824fb24493a", + "aa724824fb24493a" + ], + "comparison_refs": [ + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + }, + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + }, + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + }, + { + "concurrent": "1c044faa13e0844f", + "alone": "1c044faa13e0844f", + "serial": "1c044faa13e0844f" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 1, + "aggregate_tps": 262.4, + "per_stream_tps": 262.4, + "timing_valid": true, + "alone": { + "equal": 3, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 262.36071560784313, + "per_stream_tps": [ + 262.4 + ], + "makespan_s": 0.9929560841992497, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a957c3a797c87e88" + ], + "comparison_refs": [ + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + } + ] + }, + { + "aggregate_tps": 262.5678397430004, + "per_stream_tps": [ + 262.6 + ], + "makespan_s": 0.9921207092702389, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a957c3a797c87e88" + ], + "comparison_refs": [ + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + } + ] + }, + { + "aggregate_tps": 262.4169076295069, + "per_stream_tps": [ + 262.4 + ], + "makespan_s": 0.992701250128448, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a957c3a797c87e88" + ], + "comparison_refs": [ + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + } + ] + } + ] + }, + { + "prompt": "long-prose-chat", + "temperature": 0.0, + "streams": 4, + "aggregate_tps": 807.0, + "per_stream_tps": 206.9, + "timing_valid": true, + "alone": { + "equal": 12, + "unequal": 0, + "failed": 0 + }, + "serial": { + "equal": 1, + "unequal": 0, + "failed": 0 + }, + "reps": [ + { + "aggregate_tps": 807.0237703629164, + "per_stream_tps": [ + 212.1, + 208.5, + 201.8, + 205.1 + ], + "makespan_s": 1.2855015001259744, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a957c3a797c87e88", + "a957c3a797c87e88", + "a957c3a797c87e88", + "a957c3a797c87e88" + ], + "comparison_refs": [ + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + }, + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + }, + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + }, + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + } + ] + }, + { + "aggregate_tps": 807.952017277015, + "per_stream_tps": [ + 202.0, + 205.3, + 212.3, + 208.8 + ], + "makespan_s": 1.2841687076725066, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a957c3a797c87e88", + "a957c3a797c87e88", + "a957c3a797c87e88", + "a957c3a797c87e88" + ], + "comparison_refs": [ + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + }, + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + }, + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + }, + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + } + ] + }, + { + "aggregate_tps": 806.8641194473256, + "per_stream_tps": [ + 212.0, + 205.1, + 208.5, + 201.8 + ], + "makespan_s": 1.285506083164364, + "timing_valid": true, + "failed": 0, + "unmeasured": 0, + "evidence_refs": [ + "a957c3a797c87e88", + "a957c3a797c87e88", + "a957c3a797c87e88", + "a957c3a797c87e88" + ], + "comparison_refs": [ + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + }, + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + }, + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + }, + { + "concurrent": "c7ecfae5d841764c", + "alone": "c7ecfae5d841764c", + "serial": "c7ecfae5d841764c" + } + ] + } + ] + } + ] + } + ], + "deduplicated_output_evidence": { + "0c826dd2b65f7d4a": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "33e0fb803d6e", + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1" + }, + "83aca5c0d9d2501c": { + "tokens": 256, + "token_sha256": "33e0fb803d6e4a6babf3cf4edb9e8ed780bdf80b58cad333cb984ae7ea79f23f", + "output_sha256": "09d062e7b75448bb7b8eea7a7d4ca46ad37575eec52b00bdba74ff925a0ae5d1", + "complete": true, + "error": null + }, + "096c4e95759af816": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "9413a36cb8d4", + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + }, + "b11dff03a9d60648": { + "tokens": 256, + "token_sha256": "9413a36cb8d4af01d4196b4476b66ee97fecc28376e2615404d2367210668232", + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "95cfa5b83dc42bc9": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "2acb42920e45eb8b8048e4fac1167a60dea14112244ff37fa4465f51ff66cd69" + }, + "80f3f3ef627d7207": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "2acb42920e45eb8b8048e4fac1167a60dea14112244ff37fa4465f51ff66cd69", + "complete": true, + "error": null + }, + "ce7480621f864d70": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "861ba93f9d4752f8cfc26b981a8428d0db4f2d30729fca317935d9aefcb04d31" + }, + "87ec47f1bb1fecea": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "f30c4fcc66bb085cd245495466aed8b6f4cc69bc70f9130228b494fb14692c0b" + }, + "941c5e009cd44c4e": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "861ba93f9d4752f8cfc26b981a8428d0db4f2d30729fca317935d9aefcb04d31", + "complete": true, + "error": null + }, + "5e308b1934cc926e": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "f30c4fcc66bb085cd245495466aed8b6f4cc69bc70f9130228b494fb14692c0b", + "complete": true, + "error": null + }, + "312e01d99328eead": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773" + }, + "61b9f6421abd73f2": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "d74a8d3c98a7e24a425f434e3c70755b602bf73e2ff23fa6cdea015fb3b0a773", + "complete": true, + "error": null + }, + "0bb42460b21e44af": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "36a1d215acfbcb2d87c84b7c0bcfdd3e431da4dd968036dadeb7b6e407e97815" + }, + "4180ea3b03acb5e5": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "58d17db98fceebac84d1613c184da9ce82448ace7b23da6382acfd7098a947e5" + }, + "bec6286bf80c50f0": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "36a1d215acfbcb2d87c84b7c0bcfdd3e431da4dd968036dadeb7b6e407e97815", + "complete": true, + "error": null + }, + "1f45bd085a7d2945": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "58d17db98fceebac84d1613c184da9ce82448ace7b23da6382acfd7098a947e5", + "complete": true, + "error": null + }, + "62099b9da36ffa99": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "47d2eec82471e59cfaeee81c102987be9d4e9f756e8f776f4d7d15f3495187f8" + }, + "354f62312ff31951": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "3eb59d87c7f69b26cdbe4a758521e66b9c5e7af5695f594e4f21e1b3568a41a5" + }, + "8cbc42429afcb8be": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "47d2eec82471e59cfaeee81c102987be9d4e9f756e8f776f4d7d15f3495187f8", + "complete": true, + "error": null + }, + "0de6688c1efa0d81": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "3eb59d87c7f69b26cdbe4a758521e66b9c5e7af5695f594e4f21e1b3568a41a5", + "complete": true, + "error": null + }, + "2b2419c32c37f85c": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "25caf9d49f1b", + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955" + }, + "0f6bb87696635e5a": { + "tokens": 256, + "token_sha256": "25caf9d49f1b8e4bc3b0957fa1e6965d9052591e865cedb535312f9e8f3ac0e6", + "output_sha256": "7013268d366b7dd411fa5953c15d17d957257214e1edf70186874b9d587ba955", + "complete": true, + "error": null + }, + "bf04e149adfd45bd": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "4e5f56bf87b6", + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627" + }, + "370dacf7affd1747": { + "tokens": 256, + "token_sha256": "4e5f56bf87b6a8df340558e91733b4d9bc5a1123f34f7bf385e1f50e9f2fda23", + "output_sha256": "4e38ed8cbc80e71324f048639a99636db6a0c6b2bbea3dcc828ef2cb7dde0627", + "complete": true, + "error": null + }, + "6b9813fdb6f1119f": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "46a5c50d8ea4d0e89805969b4d818de47e1ca0943bc8f6b63248ed4e6be47ad4" + }, + "7fb3555bf55069a6": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "46a5c50d8ea4d0e89805969b4d818de47e1ca0943bc8f6b63248ed4e6be47ad4", + "complete": true, + "error": null + }, + "a3b4229cb8c41277": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "feaa37c2b89ff1af70ccf2ea98b7c8785133038c2a862c0cf8a7514e66ca6ee3" + }, + "4e2ce2be59af5602": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "f109defe08d7743ac95fae3d57a600d065dc37088301dd61b76a946176c20c69" + }, + "53a7029fb9ac15d0": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "feaa37c2b89ff1af70ccf2ea98b7c8785133038c2a862c0cf8a7514e66ca6ee3", + "complete": true, + "error": null + }, + "260f5a902140f8fe": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "f109defe08d7743ac95fae3d57a600d065dc37088301dd61b76a946176c20c69", + "complete": true, + "error": null + }, + "9c6e8cc4553be330": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "63a2be4ed8a8b152de59da48ae29e10f3db4de57316d186c89c54f5b24551c57" + }, + "47d6debc7dedd30c": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "63a2be4ed8a8b152de59da48ae29e10f3db4de57316d186c89c54f5b24551c57", + "complete": true, + "error": null + }, + "c7bc7fd07326540b": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "cce3d09b123738b0b8e4e4f55beaf42bb6e17ed1a0ddd2136a87a016420e0971" + }, + "5898795258a4c16c": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "cce3d09b123738b0b8e4e4f55beaf42bb6e17ed1a0ddd2136a87a016420e0971", + "complete": true, + "error": null + }, + "ff5c2a57d0d6ba68": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "6e82edd4bd96", + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde" + }, + "d22d0b8c96f314fb": { + "tokens": 256, + "token_sha256": "6e82edd4bd969b223a1fdbb4de5cc00ed20ba94fb608510a45d7bd0b16f4f2ba", + "output_sha256": "0b31d4c4a31333ff4929a809702619113f74a2e28edde825d87f14bf31537bde", + "complete": true, + "error": null + }, + "0c67af3b924400b5": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "d3213d33b10f", + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d" + }, + "7fedd0adca47803e": { + "tokens": 256, + "token_sha256": "d3213d33b10f84938470c3e239cfdcdeb36c4d5051991344bbe5f6087c3bd318", + "output_sha256": "68d2e2d51a70a1cd84852c0bf641f3b7f393ecbe9e028427b65ae2a824baac6d", + "complete": true, + "error": null + }, + "47e27a882f6b6bc0": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "287181355b709d4fa88ceea7f9df808451e2a85de2f33541cd350619b8fdc4db" + }, + "44111736ac335976": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "287181355b709d4fa88ceea7f9df808451e2a85de2f33541cd350619b8fdc4db", + "complete": true, + "error": null + }, + "a5a6c16baa858220": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "f31d2a54afa11a9a739ad8a4cfeb2984d3724392e8ebf4b1f41d755a2a4998b3" + }, + "53ab6a9ba0132b24": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "021f28600cc8b366f0b0d6d06b2523d2d476978e7a82bfeb14298f820323b60c" + }, + "4ff40deab3b3ee42": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "f31d2a54afa11a9a739ad8a4cfeb2984d3724392e8ebf4b1f41d755a2a4998b3", + "complete": true, + "error": null + }, + "d77d5b2340f0e3d8": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "021f28600cc8b366f0b0d6d06b2523d2d476978e7a82bfeb14298f820323b60c", + "complete": true, + "error": null + }, + "78d0d48406b30e24": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "1ad92991b86c2803dbada765356db0bba6488a52dfb4372aeb8a8dd6d09017af" + }, + "f7c90614f10021f0": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "1ad92991b86c2803dbada765356db0bba6488a52dfb4372aeb8a8dd6d09017af", + "complete": true, + "error": null + }, + "a36c6a6fd4f53222": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "c74d54c61041441ab4b794e9c8af4271969b723bc1eceeb0ea8d00bf2eec1f60" + }, + "79c7bb374559ff06": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "9dfa53730911dc2bc4e4aacfa8761dde28da710f897cea892edceff92040d8c8" + }, + "8bab9469ba3b5430": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "c74d54c61041441ab4b794e9c8af4271969b723bc1eceeb0ea8d00bf2eec1f60", + "complete": true, + "error": null + }, + "87e8945794a1bed6": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "9dfa53730911dc2bc4e4aacfa8761dde28da710f897cea892edceff92040d8c8", + "complete": true, + "error": null + }, + "aa724824fb24493a": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "8fec22df0fa1", + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + }, + "1c044faa13e0844f": { + "tokens": 256, + "token_sha256": "8fec22df0fa103bff4d7d00b83bbe2a662a354682f463e3837852822ec92ff7e", + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "a957c3a797c87e88": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": "ac404b42dea9", + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c" + }, + "c7ecfae5d841764c": { + "tokens": 256, + "token_sha256": "ac404b42dea9d714778b0b2f05d5a1b850601dd16f290d0509e61f8007618220", + "output_sha256": "14bfea24195a9416808ae7d3f0a450b3db3aa5695d61223de991f3b477faa87c", + "complete": true, + "error": null + }, + "15b30bd9b33e5220": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d" + }, + "ffff87b6ee89dd36": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "2b8e15cdba9946613e29fc8761002d8e14df5c1c8dfac347e27377a47a8f7a0d", + "complete": true, + "error": null + }, + "1c2c166eae95d7e6": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "6dcf1fea36dde63b16b8979e296a35d61fa406958b2411267275674e34654602" + }, + "27e6e32ca698e6d7": { + "prompt_tokens": 53, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "ed6bc26bfcf78fa013de555863f7f44cb91c7b9f3e88ad9f622637ce5936f724" + }, + "4be982359420377d": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "6dcf1fea36dde63b16b8979e296a35d61fa406958b2411267275674e34654602", + "complete": true, + "error": null + }, + "b1473411bdc7c6a4": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "ed6bc26bfcf78fa013de555863f7f44cb91c7b9f3e88ad9f622637ce5936f724", + "complete": true, + "error": null + }, + "6388028581e2b97b": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "aa1580092ce8b46dc61bd425d5697c7031f2e7e503428df48248be6c514babdf" + }, + "75a9f00ebfc68dba": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "aa1580092ce8b46dc61bd425d5697c7031f2e7e503428df48248be6c514babdf", + "complete": true, + "error": null + }, + "02d9b2248a4ea3f5": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "422108b9935d8eda3ede2e626504de1a6145fade72ec58008187e6ddd4f2d9b3" + }, + "01c067355e9065bb": { + "prompt_tokens": 54, + "cached_tokens": 0, + "tokens": 256, + "cache_state": "cold", + "complete": true, + "token_sha": null, + "token_sha256": null, + "output_sha256": "92f645b37b7674a6f5dee7f07ec3b8681778d25d244c6a0330e72690be832634" + }, + "8d184bece38566a4": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "422108b9935d8eda3ede2e626504de1a6145fade72ec58008187e6ddd4f2d9b3", + "complete": true, + "error": null + }, + "c64a0f957d03e6e7": { + "tokens": 256, + "token_sha256": null, + "output_sha256": "92f645b37b7674a6f5dee7f07ec3b8681778d25d244c6a0330e72690be832634", + "complete": true, + "error": null + } + } + }, + "limitations": [ + "Only M5 hardware and6/8bit2k/8k measured here;4bit is retained in its separate current-public receipt. No32k/64k,128k or native-window performance claim.", + "VL requests contain text only; real-image correctness is separate evidence.", + "Standard omits token-id fingerprints; its token equality is unverified, and output-text digests are not substituted. Native token-equality checks passed.", + "Native and standard use request budget70000, but reserved native KV and lazy standard KV/sliding allocations differ.", + "Peak physical footprint and maximum RSS are process measurements, not inferred device/pinned allocator peaks.", + "Mirrored runs retain per-leg drift; no measured thermal/frequency/bandwidth attribution.", + "Performance receipts do not establish a model FP64 fidelity pass or remove disclosed top-token differences." + ], + "raw_file_manifest": { + "events.json": { + "bytes": 49479, + "sha256": "1e55cb246ab1a054a9ef01c1900d40cb991a50af565e7a172daf79eec39d4abb" + }, + "manifest.json": { + "bytes": 17121, + "sha256": "3461a2cfcb2d32a10d961dd3d0620b212f4aad435d9f5326d441d7a8ab78e8f8" + }, + "records.json": { + "bytes": 17057, + "sha256": "4c2efff58789078dbf5ed5bb74597d93c98bafca556f8a01a06e85d2aec022ff" + }, + "text-6-0-current.after.health": { + "bytes": 303, + "sha256": "a9616be96809061e248add31809cfad79caf07dc1518a7c75f2615e0ffbc11d3" + }, + "text-6-0-current.after.metrics": { + "bytes": 12298, + "sha256": "42127e0babeeb70e345bf58118f60e1ebdab01e84c8979ca676d842081824b7c" + }, + "text-6-0-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-6-0-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "text-6-0-current.before.metrics": { + "bytes": 12128, + "sha256": "80b4086c88850ea24dcb1891a3854774f6f4ff1d69a52474eea6cfa328847789" + }, + "text-6-0-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-6-0-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "2b74b2084920c0be5b369a12bc6afc54ab249d31a0cb625f9ee8c9fe4972a9ac" + }, + "text-6-0-current.cold-2048.json": { + "bytes": 20277, + "sha256": "ebbb0bdc92ccf32fb7d24fa62cc28021efb1dc45f5eb7e5156601de93bce34a0" + }, + "text-6-0-current.cold-8192.client.log": { + "bytes": 346, + "sha256": "657ab06e0dd2b4a3269cbe325ce5d583ad81d22242e12829155c65dbd4791800" + }, + "text-6-0-current.cold-8192.json": { + "bytes": 57153, + "sha256": "61cb0c43da2413ee292742931fbb61277f510e4c9e1669fcf494295514645a95" + }, + "text-6-0-current.concurrent.client.log": { + "bytes": 3221, + "sha256": "dae3db4782b6e73d3c96645947537c3743c93178404675217b5a7b72514a5099" + }, + "text-6-0-current.concurrent.json": { + "bytes": 292701, + "sha256": "36a383f37cfb7b4b4879c1d8521f9e5cd8e4e39a8578835acaa3aa5155519b6a" + }, + "text-6-0-current.server.log": { + "bytes": 13634, + "sha256": "e338f34c3fe8bbbc019f9a73375d3cb2290ac6193a28889cef846eee20ecc090" + }, + "text-6-1-standard.after.health": { + "bytes": 453, + "sha256": "87944493ce57beba80069a2d02208f7b9dbf8683b03ab95ad00b4bffdb7e8e2f" + }, + "text-6-1-standard.after.metrics": { + "bytes": 32126, + "sha256": "9fe294108d7f38dad73053f737f44222381161d0cafb48ad364dba5ec5133b9b" + }, + "text-6-1-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-6-1-standard.before.health": { + "bytes": 453, + "sha256": "87944493ce57beba80069a2d02208f7b9dbf8683b03ab95ad00b4bffdb7e8e2f" + }, + "text-6-1-standard.before.metrics": { + "bytes": 909, + "sha256": "4daf5f787b64b4e4279f019dcf4a297d8b8b46ea30950170ccdd1cabd86dcb26" + }, + "text-6-1-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-6-1-standard.cold-2048.client.log": { + "bytes": 297, + "sha256": "09aff08f271e4eb73bd76884e688150fca657e6d38010b70956e2e7ca7c583c0" + }, + "text-6-1-standard.cold-2048.json": { + "bytes": 26083, + "sha256": "b5bdf7b82002359f7b1c90a32bba6c5a1f0a80fe46118b730cc67f81fb49ad7d" + }, + "text-6-1-standard.cold-8192.client.log": { + "bytes": 295, + "sha256": "de88203e1d7a587072a52b1269804dee3ee79acd6cba34063b0c69d1ad6eda0b" + }, + "text-6-1-standard.cold-8192.json": { + "bytes": 62937, + "sha256": "af2079563e3371bc97bc43471f9242358be38e87137355f70a312cd576ac085f" + }, + "text-6-1-standard.concurrent.client.log": { + "bytes": 3234, + "sha256": "7f8a0dd74d7b923c35d36a797f01551e3d9373065e8221436fc2a831b53cd36b" + }, + "text-6-1-standard.concurrent.json": { + "bytes": 285286, + "sha256": "7f327cf4f2875e1d5cc8df95dea1b024f431ec6678a3853d1779cc2a019f3f9f" + }, + "text-6-1-standard.server.log": { + "bytes": 50891, + "sha256": "21df098eb5f6a2b486d0fd574855d34f890af8675c98c6ca7bb11c846a94d252" + }, + "text-6-2-standard.after.health": { + "bytes": 453, + "sha256": "87944493ce57beba80069a2d02208f7b9dbf8683b03ab95ad00b4bffdb7e8e2f" + }, + "text-6-2-standard.after.metrics": { + "bytes": 32135, + "sha256": "8a61eb4fd923a8eeae821138af8d216b306dd69b79ad784552952680e207736a" + }, + "text-6-2-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-6-2-standard.before.health": { + "bytes": 453, + "sha256": "87944493ce57beba80069a2d02208f7b9dbf8683b03ab95ad00b4bffdb7e8e2f" + }, + "text-6-2-standard.before.metrics": { + "bytes": 909, + "sha256": "988211da7bcc099a3f31f49330649e522e10ebcae345c63e767fb9e0c02f7c50" + }, + "text-6-2-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-6-2-standard.cold-2048.client.log": { + "bytes": 297, + "sha256": "e0b77645d2fb7635574f8c05b137e6b9624d5fe5bc1212adba0d607f91975b2d" + }, + "text-6-2-standard.cold-2048.json": { + "bytes": 26082, + "sha256": "8161742b232abd7ecaab3f3f6aa2b28f30a04ac2f5b9c40d8d7fca096bce2a13" + }, + "text-6-2-standard.cold-8192.client.log": { + "bytes": 297, + "sha256": "34303db632d285f6f4b2389142a10d234c9c9bbe99304a1293a902fafd8e86dc" + }, + "text-6-2-standard.cold-8192.json": { + "bytes": 62945, + "sha256": "da2e72beaf2d0fb3cf8bd6722bfe6b09fad9c5a4987054a5a8a764355ec89e72" + }, + "text-6-2-standard.concurrent.client.log": { + "bytes": 3232, + "sha256": "f76f1e9c08747727cbce7d507c88e5ab2f9a209651a70eafba1f9a718008d066" + }, + "text-6-2-standard.concurrent.json": { + "bytes": 285274, + "sha256": "bc77ae8452f2c07d2c4952974895d6c8ad38140cd6fabb4d1cf61c5f6c1ebc34" + }, + "text-6-2-standard.server.log": { + "bytes": 50891, + "sha256": "482dcca26737756163d84fd1ed1ad8bf278c311aa2e11e1e3ac570a66882ffb7" + }, + "text-6-3-current.after.health": { + "bytes": 303, + "sha256": "047f2a6a99c470c6b7c69a4e5c18cd4f91cca1389f3a96f3fbaa54b01997846b" + }, + "text-6-3-current.after.metrics": { + "bytes": 12298, + "sha256": "62293f3de70dd01adc1591ccf0a4cf73d1a314f3dc3cc48471d3109707df7093" + }, + "text-6-3-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-6-3-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "text-6-3-current.before.metrics": { + "bytes": 12128, + "sha256": "4f61b39294688798540c9efe9cc437c0af1fe9e01fbe0ac1c9b86f96290e005f" + }, + "text-6-3-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-6-3-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "58503dcf48365f99574d308275ba4fa8ebd0bb731b16967496f98050768f7ab0" + }, + "text-6-3-current.cold-2048.json": { + "bytes": 20281, + "sha256": "f271a7ca06400ca2e7cbc9b2c9531b4be80ef039b310d014dbf7b14b377eb458" + }, + "text-6-3-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "9551cc002427b94ae82cacbca7c67c04c7b668f591430ca3272a71ac1a1a1868" + }, + "text-6-3-current.cold-8192.json": { + "bytes": 57157, + "sha256": "2f407f58dbbe86b7aae79893d103f6a64fa7c621b013594cf0e52a15f529c51c" + }, + "text-6-3-current.concurrent.client.log": { + "bytes": 3223, + "sha256": "33823c5f8eeb39cba0f7c3a293d168e49844684f7a882a54138e8c53fbd3a010" + }, + "text-6-3-current.concurrent.json": { + "bytes": 292700, + "sha256": "b646e91b78dda6d5055d29e6c29ce9838a5f299c0ac472954e7a2206d75a5e31" + }, + "text-6-3-current.server.log": { + "bytes": 13634, + "sha256": "28f05be106cc010088beaff8f4a4d433ef129cdbeac20ceb3c5a0acf252000ee" + }, + "text-8-0-current.after.health": { + "bytes": 303, + "sha256": "ec3ee92ddbeb3d245a2f8920eef138b8c05e0cffd45e649de196c5be1f1a4467" + }, + "text-8-0-current.after.metrics": { + "bytes": 12297, + "sha256": "c13ccdbc96d1064f4b2856ea0b5c8d2c7a1e5979cb671f505694337d18930dfd" + }, + "text-8-0-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-8-0-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "text-8-0-current.before.metrics": { + "bytes": 12128, + "sha256": "f3c64235a7216dd0d57beea4a549b4f5328f3216c43259354de19b85438c9c1a" + }, + "text-8-0-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-8-0-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "e037e75904d53c65e438808c12ee57453ca78eb466ec7c9d13920023f9358ccc" + }, + "text-8-0-current.cold-2048.json": { + "bytes": 20295, + "sha256": "430c3c0fb2f7ecf1085c96117ccf82f1039c04b696b224efd3bf5896394c0fe7" + }, + "text-8-0-current.cold-8192.client.log": { + "bytes": 346, + "sha256": "777d5f52ba839dce0629e86026e07a2ce18cd55a391d54db6b7006d90d9ac0ff" + }, + "text-8-0-current.cold-8192.json": { + "bytes": 57143, + "sha256": "20ec44def5324c7e04d07b01fa51fda88e6ede8cd352900ff6ef6caf31ce1e74" + }, + "text-8-0-current.concurrent.client.log": { + "bytes": 3227, + "sha256": "b503c3295a5f4bf60e2900886b65ada637dadded49d9a4224ba8d281a1585d5a" + }, + "text-8-0-current.concurrent.json": { + "bytes": 292830, + "sha256": "adfcc6d543483f3fa0dba6aef9b862a9d6c076fcfaf22ecdf059f2a16ca350be" + }, + "text-8-0-current.server.log": { + "bytes": 13637, + "sha256": "7eb96a88a9973bcb9c2fb54af1cb213bae6172918b14e4861e0c7634323dc722" + }, + "text-8-1-standard.after.health": { + "bytes": 453, + "sha256": "c6a2a7b6cbe67290e76f6275e124f57a57c5d2287f34843e2cbbc535bf9d6671" + }, + "text-8-1-standard.after.metrics": { + "bytes": 32122, + "sha256": "fb68a7a8cace89de3649d36071ff38a0fd0e9f6005b597280089ff16d6933ce8" + }, + "text-8-1-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-8-1-standard.before.health": { + "bytes": 453, + "sha256": "c6a2a7b6cbe67290e76f6275e124f57a57c5d2287f34843e2cbbc535bf9d6671" + }, + "text-8-1-standard.before.metrics": { + "bytes": 908, + "sha256": "ffb4ba8f198d74b923641f3adf8c81fdbda006488c86cd35c1457f1d52551b47" + }, + "text-8-1-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-8-1-standard.cold-2048.client.log": { + "bytes": 295, + "sha256": "310945f7f91fd9107494a77fb89ec56c9e3a671d43b28915011997bd8aeb4a31" + }, + "text-8-1-standard.cold-2048.json": { + "bytes": 26076, + "sha256": "fec629a2bf278fcac13443d520ac50012c9d64474215af8e780ed7ed8bb56fd0" + }, + "text-8-1-standard.cold-8192.client.log": { + "bytes": 295, + "sha256": "f927d951ace3882a0bea7b86d5d2513597bf4e75d023bb459a131a0a94c6671a" + }, + "text-8-1-standard.cold-8192.json": { + "bytes": 62930, + "sha256": "27759fc13fe80b50283c260c6272939297e62eb62fcc2e80fdfaa8725293697c" + }, + "text-8-1-standard.concurrent.client.log": { + "bytes": 3229, + "sha256": "803985e8713a012e55828b1f87a05bfe1732314a1833c686e69d3dda9389bd2e" + }, + "text-8-1-standard.concurrent.json": { + "bytes": 285270, + "sha256": "b4bb0b9701ee62304a717fcedac25deeb9d2fdb5813f983ca44058416760e7db" + }, + "text-8-1-standard.server.log": { + "bytes": 50891, + "sha256": "a7f891f2fe6c4544363ec2df1b463ef41c6ef50296a1eda8b855269915161f1a" + }, + "text-8-2-standard.after.health": { + "bytes": 453, + "sha256": "c6a2a7b6cbe67290e76f6275e124f57a57c5d2287f34843e2cbbc535bf9d6671" + }, + "text-8-2-standard.after.metrics": { + "bytes": 32114, + "sha256": "7aefca52488c20bb1f0db32a7162f82d8311fe5b44764a51ba4ecba623b5924a" + }, + "text-8-2-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-8-2-standard.before.health": { + "bytes": 453, + "sha256": "c6a2a7b6cbe67290e76f6275e124f57a57c5d2287f34843e2cbbc535bf9d6671" + }, + "text-8-2-standard.before.metrics": { + "bytes": 908, + "sha256": "526a57fe3d5be98e32fd8141c5ccdbee098e46ab6f525746780352142a81af1c" + }, + "text-8-2-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "text-8-2-standard.cold-2048.client.log": { + "bytes": 296, + "sha256": "a15252ef5353e2805633558d943cbce05e72c151417f20e907fc0ae755c899c4" + }, + "text-8-2-standard.cold-2048.json": { + "bytes": 26074, + "sha256": "9626b4b94b4358dcf1393f6b5ca5ccdf5f9e1a77d1f94d1079c99a125ec100df" + }, + "text-8-2-standard.cold-8192.client.log": { + "bytes": 296, + "sha256": "0091e6a0ecc30f8637652ffca697125ad0354ff225f4c8bd0bb93ef09c8f4162" + }, + "text-8-2-standard.cold-8192.json": { + "bytes": 62932, + "sha256": "7f714759b65f0b84ddf76a86e2de622b601a0bc69f321810f6266e8d2dba8991" + }, + "text-8-2-standard.concurrent.client.log": { + "bytes": 3226, + "sha256": "57fe99245159f35425ab66b15cbfc254ddb2f2c93dd4c03cee8dd91f885e3893" + }, + "text-8-2-standard.concurrent.json": { + "bytes": 285340, + "sha256": "fa942f2f3291a1f859154d1d23c4af030bccd72e09b1f485a42334de6d2a5cee" + }, + "text-8-2-standard.server.log": { + "bytes": 50891, + "sha256": "887e63589861a355c26af6aaa3af66c67acf5418425cb774b155511748bf0af7" + }, + "text-8-3-current.after.health": { + "bytes": 303, + "sha256": "7d041e37775a9ec72201faed8a37222e2182010675c273c5e0ae12bf66a2fc4b" + }, + "text-8-3-current.after.metrics": { + "bytes": 12297, + "sha256": "e6077dd7a6b59fdad46611a5577f21dd468e118becb764b37561c7320f8f30d2" + }, + "text-8-3-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-8-3-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "text-8-3-current.before.metrics": { + "bytes": 12128, + "sha256": "05e17ede1e0b6a9abaa950940c9aecfc5e0752a4a6f7fed3c942e3bc786fe85d" + }, + "text-8-3-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "text-8-3-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "e26a3083742a074fe04f06db81d8b1018a0eabbf3dff2a8880f79f771eadd012" + }, + "text-8-3-current.cold-2048.json": { + "bytes": 20295, + "sha256": "a6cdcd146244875f12c29aa65338bab3ed0f0242d4f2902fb9fa744afe79c1c7" + }, + "text-8-3-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "afe30938da4000bd69f6f8b5d806d418e097acc2662e0092786d7673b7e28a5f" + }, + "text-8-3-current.cold-8192.json": { + "bytes": 57144, + "sha256": "9d5a7e9a378ac1fe1c62b7c7b4d9f13ebadb70858ed797618fe048129796c557" + }, + "text-8-3-current.concurrent.client.log": { + "bytes": 3210, + "sha256": "30d1f636fc782236ff4493822d29df3216027074f06bb4755467d4ce292bb17f" + }, + "text-8-3-current.concurrent.json": { + "bytes": 292731, + "sha256": "84d1615396089cbf0589710963fc2dc3905551e676a9cc0ce13302d4bd2a9e09" + }, + "text-8-3-current.server.log": { + "bytes": 13637, + "sha256": "226343ca4b671ffe3785239ca0ab6dbae6a131c1ff7e37bed05ab30dfe5d7148" + }, + "vl-6-0-current.after.health": { + "bytes": 303, + "sha256": "27092cec238d7c761b5e8453789b995e0514416331bf78f14744f4aeb5d415b0" + }, + "vl-6-0-current.after.metrics": { + "bytes": 12306, + "sha256": "829ad01b034ec790a696168e9ca70ba325fca0303125faede4de51c297f0a43f" + }, + "vl-6-0-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-6-0-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "vl-6-0-current.before.metrics": { + "bytes": 12128, + "sha256": "1654eba5ff03499ae49fe04ed23d5571f36d03df5250442533e53bacaacb4a06" + }, + "vl-6-0-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-6-0-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "8be7e8e09abe7c82f81d31e7be1dc530e4144d567bec27d60c407c49f19abd1d" + }, + "vl-6-0-current.cold-2048.json": { + "bytes": 20279, + "sha256": "011dc52417e9c3f08bd6d0d736fd7fee472611ada26d452091f105bba712ca6a" + }, + "vl-6-0-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "c09af7ce9d1abe7b85379b4f0c979fc43d6857a3d1a2c24f726bcc4982236a35" + }, + "vl-6-0-current.cold-8192.json": { + "bytes": 57145, + "sha256": "c550257dba077a7c25962448c51d033f946be27425823a23105afbfc98c7cb9b" + }, + "vl-6-0-current.concurrent.client.log": { + "bytes": 3230, + "sha256": "34c37d77798060df3c38463509c9ccc5cc17ad518e95149d41b8f7b4f99a838a" + }, + "vl-6-0-current.concurrent.json": { + "bytes": 290243, + "sha256": "912ba7e96f1638c00ee1e1456014517b11fdd0263e61bdd04f1fad464a46f219" + }, + "vl-6-0-current.server.log": { + "bytes": 13642, + "sha256": "5fb5b4339ddf144e557f0df6735b058522c27094d5a0252f7d31597290842e0d" + }, + "vl-6-1-standard.after.health": { + "bytes": 457, + "sha256": "d6e561acb183d0bff082d53d5cb1ad0d93a54531ac634fb00bccda2363411ab8" + }, + "vl-6-1-standard.after.metrics": { + "bytes": 32168, + "sha256": "99e97844a2d9dcbf39764232aff5b1db2b6d20e5130e03782c4835350b5e7ea5" + }, + "vl-6-1-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-6-1-standard.before.health": { + "bytes": 457, + "sha256": "d6e561acb183d0bff082d53d5cb1ad0d93a54531ac634fb00bccda2363411ab8" + }, + "vl-6-1-standard.before.metrics": { + "bytes": 913, + "sha256": "70426c7a4b86e9921db23ba4b0033e566817c59f37835e551a11d6de9c721a1a" + }, + "vl-6-1-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-6-1-standard.cold-2048.client.log": { + "bytes": 297, + "sha256": "b241f0cf7544beea964b47747da23e732d8dccad196089941d719fd50c6849fd" + }, + "vl-6-1-standard.cold-2048.json": { + "bytes": 26077, + "sha256": "ca67f9ecf9607e2ac2ebd2b46075fbde3e548eb3f3d87c5eadf1814c5187388e" + }, + "vl-6-1-standard.cold-8192.client.log": { + "bytes": 297, + "sha256": "146a875633757bfc0617563c15f917bac2571c07e54eeaaf145bf611ca6a4701" + }, + "vl-6-1-standard.cold-8192.json": { + "bytes": 62951, + "sha256": "c2319dd650b4feb05e0ab736856feb21b460e4955e6a3c2461e798ae875ff242" + }, + "vl-6-1-standard.concurrent.client.log": { + "bytes": 3222, + "sha256": "f052af60311c5900bfabd2c857e03b91d5b91fe90599c87aaa19e7c36c6ebe43" + }, + "vl-6-1-standard.concurrent.json": { + "bytes": 285398, + "sha256": "d92d0c1d0378bff888fee108bc6011d790ccc81193bcbf39aafcc6f0a24c0d7f" + }, + "vl-6-1-standard.server.log": { + "bytes": 51083, + "sha256": "b6ac9019fc1600364ebda1110d31e29b8f22447f734f5cf84c11131cd0c108e1" + }, + "vl-6-2-standard.after.health": { + "bytes": 457, + "sha256": "d6e561acb183d0bff082d53d5cb1ad0d93a54531ac634fb00bccda2363411ab8" + }, + "vl-6-2-standard.after.metrics": { + "bytes": 32184, + "sha256": "7595d894ef1d409b6bdba8c4cc49da2afb7ec6144c36b74bae0ecbe0ed5cbbff" + }, + "vl-6-2-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-6-2-standard.before.health": { + "bytes": 457, + "sha256": "d6e561acb183d0bff082d53d5cb1ad0d93a54531ac634fb00bccda2363411ab8" + }, + "vl-6-2-standard.before.metrics": { + "bytes": 912, + "sha256": "ca25563480ae3600ec7b170ab15644e9a42ef2227ffae7719c9f1e762115cb66" + }, + "vl-6-2-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-6-2-standard.cold-2048.client.log": { + "bytes": 296, + "sha256": "609e07f2788126d75a435082ee6215ef297a925db28ca062ab05a4b92913614c" + }, + "vl-6-2-standard.cold-2048.json": { + "bytes": 26078, + "sha256": "34f9c58f325ab5eaccfe3d11b49c77c834ca16062e42740e571a7a5b9111bc1b" + }, + "vl-6-2-standard.cold-8192.client.log": { + "bytes": 297, + "sha256": "0616611f1d3ad0ba8ec6604b66b5bdfb219fd81d26122b8a42ace7567ef4096b" + }, + "vl-6-2-standard.cold-8192.json": { + "bytes": 62953, + "sha256": "1b71d0d3ea2592addcdd343fc57e31a631dfc1332110a26540a0a90929d44eb7" + }, + "vl-6-2-standard.concurrent.client.log": { + "bytes": 3220, + "sha256": "050e709e808bc69668ae76e493f205161317a953d440d52462a20e0b470776fb" + }, + "vl-6-2-standard.concurrent.json": { + "bytes": 285412, + "sha256": "79921717738f9bc6c644159f0c9d1828a69e86993f207e762d1a5603fc9a6c33" + }, + "vl-6-2-standard.server.log": { + "bytes": 51119, + "sha256": "63543991ec6dc5a139891588c915c790381c79c00f2c1fe96befd02f0b1da44f" + }, + "vl-6-3-current.after.health": { + "bytes": 303, + "sha256": "81303458a4718d9abb89de41ec631ba9d702281405b7a56cd10af88fdf33ad0f" + }, + "vl-6-3-current.after.metrics": { + "bytes": 12305, + "sha256": "741c1ecc6b645550111305ea8d3ce825c6bc44c82ec5ec42346738cb1791202b" + }, + "vl-6-3-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-6-3-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "vl-6-3-current.before.metrics": { + "bytes": 12128, + "sha256": "383f94ac56bd67ca477ed6d4d34a029dff950051e4b9b7303d121f10a074f135" + }, + "vl-6-3-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-6-3-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "533815c80da6219676d15f2b4f18abc6ca736606fe3473bf73bb15bc3b935061" + }, + "vl-6-3-current.cold-2048.json": { + "bytes": 20278, + "sha256": "c2c0972f342720cc1605c84ebffbfb66b805fed0df9585f4bfa82cf610cb6c2b" + }, + "vl-6-3-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "84ea4ac7d5ccc2d1ba988083436aca44f13d604f857cf7b58314a1338b063bef" + }, + "vl-6-3-current.cold-8192.json": { + "bytes": 57145, + "sha256": "40afd4ab46755fe12a0009067bc2ef31f2b0a09cf5de8fd5700e61b7974546d9" + }, + "vl-6-3-current.concurrent.client.log": { + "bytes": 3233, + "sha256": "b794a7ef86bde67f0c3ca210504925698291e79a8567fc41a7b6e50d84a7b61f" + }, + "vl-6-3-current.concurrent.json": { + "bytes": 290300, + "sha256": "e6bf43dc03bb1fdc6cb5662a6626c3cc801091794d6567fd1365a9e277ba5cc7" + }, + "vl-6-3-current.server.log": { + "bytes": 13642, + "sha256": "31aac91a44f32e60f49a213bcae64536c2709e0d02ed9aa8a648a50a200b10c9" + }, + "vl-8-0-current.after.health": { + "bytes": 303, + "sha256": "a7b510179c027078a79ba1d1e14ac96e95d33213b7111cd873127c92327ce798" + }, + "vl-8-0-current.after.metrics": { + "bytes": 12304, + "sha256": "5813647737abb033d170a64938985ad06b4e6bbc0ec8302afc9b2b576de511ff" + }, + "vl-8-0-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-8-0-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "vl-8-0-current.before.metrics": { + "bytes": 12128, + "sha256": "301db95b2706ef8748d58c6a95b2e75d48551c9e11d6476cc6fd67010f47477f" + }, + "vl-8-0-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-8-0-current.cold-2048.client.log": { + "bytes": 348, + "sha256": "8f7f61eeeeb080a9e89d749a12e124d3318500cce659e7c07246151263d9a066" + }, + "vl-8-0-current.cold-2048.json": { + "bytes": 20276, + "sha256": "f666c81a25cdcea210246b5b01ecf450e8cdf02bf10edde69dcb1889e2cae87d" + }, + "vl-8-0-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "8cd954da93125b581780d1fdeda4aa3f3fd66c05a35e2d880bad3467e641094c" + }, + "vl-8-0-current.cold-8192.json": { + "bytes": 57141, + "sha256": "94237f24059cbc2c18ee0decc0817b4a01fa7dc815e91b3be2b8bac632706fd5" + }, + "vl-8-0-current.concurrent.client.log": { + "bytes": 3225, + "sha256": "b85243b163154075d156dca6a2e21b5527986053114fa8733d7ae9dde91ad082" + }, + "vl-8-0-current.concurrent.json": { + "bytes": 288866, + "sha256": "efbb0beb927bd45ba030170913ea780d2bd7462256f11263ca399e0705b90085" + }, + "vl-8-0-current.server.log": { + "bytes": 13641, + "sha256": "a89b78f9653448bf13d101666f8f4052ee727b6122073d9e471a9d0eb3963e60" + }, + "vl-8-1-standard.after.health": { + "bytes": 457, + "sha256": "d77972274c900ea75a7c25828e72058554a47ad134d26ab4acb86a59e04ff339" + }, + "vl-8-1-standard.after.metrics": { + "bytes": 32127, + "sha256": "8340257ed87f8940aa9d430bb48249eb5abf1aadd680aca3c4949e8bf61339e8" + }, + "vl-8-1-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-8-1-standard.before.health": { + "bytes": 457, + "sha256": "d77972274c900ea75a7c25828e72058554a47ad134d26ab4acb86a59e04ff339" + }, + "vl-8-1-standard.before.metrics": { + "bytes": 913, + "sha256": "9928f200727b8e4df574a61284adce4dc428150550b12517a834da740952b80d" + }, + "vl-8-1-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-8-1-standard.cold-2048.client.log": { + "bytes": 297, + "sha256": "2572925fb424878917803146fafd82c0686295ad5bc22578796ac11cc8fc140b" + }, + "vl-8-1-standard.cold-2048.json": { + "bytes": 26082, + "sha256": "b939e2c2e006b0b1d0f2b6155fc94fb57e68c4552aad7ee0ba78762203516741" + }, + "vl-8-1-standard.cold-8192.client.log": { + "bytes": 297, + "sha256": "7bef492d469a9a85487158880c032c329bad52b75adfa2c9e7dc04b792c13515" + }, + "vl-8-1-standard.cold-8192.json": { + "bytes": 62948, + "sha256": "da107e7322f082e989ed71edc747e9013154525ebcf8d7d356f2ba784a834dbf" + }, + "vl-8-1-standard.concurrent.client.log": { + "bytes": 3222, + "sha256": "42e9cc77c748a764d500bbe8063e04af7c00ec1a2f207cc84aded9fbef202ab9" + }, + "vl-8-1-standard.concurrent.json": { + "bytes": 285423, + "sha256": "9220f89e6ee7bbb83e06134fa6c6b5f9c64f51d073dc7b724d50dc3f151715f2" + }, + "vl-8-1-standard.server.log": { + "bytes": 51107, + "sha256": "6a75266b138a78b383f54e20ecface5feda4976a8d093b4678f80510c2813523" + }, + "vl-8-2-standard.after.health": { + "bytes": 457, + "sha256": "d77972274c900ea75a7c25828e72058554a47ad134d26ab4acb86a59e04ff339" + }, + "vl-8-2-standard.after.metrics": { + "bytes": 32129, + "sha256": "e51ced6f682aeaeb777422035293ad4549b3975c56711ee3964a0fe00c72b25f" + }, + "vl-8-2-standard.after.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-8-2-standard.before.health": { + "bytes": 457, + "sha256": "d77972274c900ea75a7c25828e72058554a47ad134d26ab4acb86a59e04ff339" + }, + "vl-8-2-standard.before.metrics": { + "bytes": 913, + "sha256": "dae43aa6527a30ee6197dd8b93a22e68df989f0814509b42da5b72fe2ce07b39" + }, + "vl-8-2-standard.before.v1-settings": { + "bytes": 5417, + "sha256": "384b681408158ae95746f7f862408db3faff798130d2664be259654ee5c4d4ed" + }, + "vl-8-2-standard.cold-2048.client.log": { + "bytes": 297, + "sha256": "c76ab5799f24c678f15504fd1782bbaec0ea99e9e6322b9781132f9e3844ca74" + }, + "vl-8-2-standard.cold-2048.json": { + "bytes": 26082, + "sha256": "3b4d451fa5b93dc2ce412e1e624d0f21f8e3c53fa5dd87c2e099abeef52af397" + }, + "vl-8-2-standard.cold-8192.client.log": { + "bytes": 295, + "sha256": "9ab2a6d260383bae8035ceccb1e4f6f5ac56d5c9fe3802cba32097b8a54e0e94" + }, + "vl-8-2-standard.cold-8192.json": { + "bytes": 62943, + "sha256": "e053e787d01d3f26aed84f37612f465d6c1ddd1104e3f94ebb390ceec5558ac0" + }, + "vl-8-2-standard.concurrent.client.log": { + "bytes": 3225, + "sha256": "48eb513e02d146767a7f14511c0f59c35774a3f8e6edba4e2b3c1916a5563f2b" + }, + "vl-8-2-standard.concurrent.json": { + "bytes": 285287, + "sha256": "804f1992d6a3ba4e2374d8dfcea874051945fbc578610214fd126790a8490abf" + }, + "vl-8-2-standard.server.log": { + "bytes": 51125, + "sha256": "0de141346dc31c081e8826aa34fd0f7368e4b24818940f60e2c6ea8f69c07e09" + }, + "vl-8-3-current.after.health": { + "bytes": 303, + "sha256": "5df60fcd4d7708232d8ae02e246efe009d332f0d57e3b9caca24da59642a76e7" + }, + "vl-8-3-current.after.metrics": { + "bytes": 12304, + "sha256": "5a059e416184dfe60c6b2859c6f2754340b3d3218835d8615d255e70d7851a82" + }, + "vl-8-3-current.after.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-8-3-current.before.health": { + "bytes": 298, + "sha256": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599" + }, + "vl-8-3-current.before.metrics": { + "bytes": 12128, + "sha256": "e1d8ba708e061e938244dabc86a9e46659fbb9b72c9aa13155ce2ed175fc3f5e" + }, + "vl-8-3-current.before.v1-settings.unavailable": { + "bytes": 9, + "sha256": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5" + }, + "vl-8-3-current.cold-2048.client.log": { + "bytes": 349, + "sha256": "770e170bd841b964b8e09a3be820c1dee1ae817938722f2e587fa8816f3d310f" + }, + "vl-8-3-current.cold-2048.json": { + "bytes": 20279, + "sha256": "c61a88909900ffdcc9236362342e0b4ba54cf0ca6b14f329614f3f48acf0c7f1" + }, + "vl-8-3-current.cold-8192.client.log": { + "bytes": 347, + "sha256": "67b7266943b354ce74059abf755e9d067d943a299813c2480a9734f89f26d165" + }, + "vl-8-3-current.cold-8192.json": { + "bytes": 57140, + "sha256": "5ff41c6c9ead50864385713c36790645a5d737852981a630d56f7a81b840e90e" + }, + "vl-8-3-current.concurrent.client.log": { + "bytes": 3228, + "sha256": "1c069b5bb6676f88489a66061569ca6b55c4768c0bd2dfed49caa538aa20c0b9" + }, + "vl-8-3-current.concurrent.json": { + "bytes": 288853, + "sha256": "41799d6c8f51105a0d998dc4fcef6a16ccd825402e8be7631b878e488c8f092b" + }, + "vl-8-3-current.server.log": { + "bytes": 13641, + "sha256": "7fc3fd5c630e654744bfd4e13f7f59c51f3fcef770a261cb626672c071517aec" + } + }, + "raw_retention": "Immutable raw HTTP results, logs, environment and manifests retained privately; relative names and SHA256 supplied. Private paths and request IDs omitted from publication." +} diff --git a/docs/receipts/lfm3/m5-prefill-batching.json b/docs/receipts/lfm3/m5-prefill-batching.json new file mode 100644 index 0000000000..2e0bbf3db8 --- /dev/null +++ b/docs/receipts/lfm3/m5-prefill-batching.json @@ -0,0 +1,441 @@ +{ + "scope": "Private M5 text4 batching-only causal screen; not enabled publicly and no all-width/final standard performance acceptance", + "change": "Only local native compute chunk and LaneHost prefill_step128/512/1024; globalcacheRules chunk128/repeat_min256 unchanged; all numerical kernels unchanged", + "builds": [ + { + "chunk": 128, + "native_source_sha256": "defb0b6ef8eee2fa72015575c70ae007cfb182b03a6ca73011cedc1082b08d13", + "binary_sha256": "9bee163d44bb3f2aa27d6d06fa791734e043d403a08f48b1dc3ba313cc4fd2c9", + "cacheRules_prompt_chunk_unchanged": 128, + "changed_compute_and_LaneHost_prefill_step": true + }, + { + "chunk": 512, + "native_source_sha256": "0ee5614230af82f626f45d81dab48271e18b329eeab9a2e9423d6a50abb967de", + "binary_sha256": "ed1b7601d65d520822f4208365d389ffe5e10353e8377932da76338442c849db", + "cacheRules_prompt_chunk_unchanged": 128, + "changed_compute_and_LaneHost_prefill_step": true + }, + { + "chunk": 1024, + "native_source_sha256": "d5360c03c851878169e04e1530126e05d248dd8c3a77e6768a7c4c6458d68df8", + "binary_sha256": "72ac6ca09394400a0d2afe6a6d27a40245ca93e8c57d2758bc742744c39aa3e3", + "cacheRules_prompt_chunk_unchanged": 128, + "changed_compute_and_LaneHost_prefill_step": true + } + ], + "environment": { + "host": "ProductName:\t\tmacOS\nProductVersion:\t\t27.0\nBuildVersion:\t\t26A428\n", + "physical_memory_bytes": 274877906944, + "requirements_sha256": "275b369ea72d5757b1e3f91e68ed5b746d83b5f7667adc8f5d9d28bb8933ddd6", + "clients": { + "$PACKET/tools/bench_auth.py": "1896120b70d0df05646870a252697e20ce84c63e09dcba88b2a4e5a55ccb3c5b", + "$PACKET/tools/bench_cache.py": "24364dd13c033450f45710fc8685e2c64d6a72d6dd6e223d0d7b19aabbdc1d98", + "$PACKET/tools/bench_concurrent.py": "0e72e26769f082459329fc068c25f6f72a9155e422962ea54cb5d1226227752d", + "$PACKET/tools/bench_evidence.py": "54c28474f0804dbd120e3fb5859be0d1667ef21ca502880d0bed098ec9bb42ff", + "$PACKET/tools/bench_metrics.py": "26f4c3ecbb03df4f12e24d18840fea989d9ef8c63117ca11eb944ef670458a27", + "$PACKET/tools/bench_openai.py": "d5af4fbb756cdfb573546c0e0017652a4bb8f509d1251af683be7a3cbdd426f6", + "$PACKET/tools/bench_timing.py": "158f636ef0b3bed460ce19ef4872b6d8607ff7a8590562d06bae8bd55e6f44ed", + "$PACKET/tools/lfm3/bench_decode.py": "c011106f4f182db19371e3f91f70bcc374160786dd105f2d713a7fea07ab8c4c", + "$PACKET/cold_client.py": "7b8ea55aa88834814f95a6f679654e16074b7a8364e34b39a276f009fe0231f2" + } + }, + "numerical_proof": { + "cases": 12, + "all_valid_KV_convstate_and_logits_exact_across_chunks": true, + "all_plain_profiled_state_and_logits_exact": true, + "timing_fields": "DISCARDED for entire numerical-only proof because ownedCPUcompilation overlapped; no plain,profiled,counter orGPUclock fromthatproof used forcost. Fresh guardedHTTPscreen supplied measuredclocks.", + "identities": [ + { + "variant": "current", + "tokens": 2048, + "chunk": 128, + "state_sha256": "7f7e23060107d5252391778f46727022f8ee157fa34eb1753c424cac2f3f771f", + "logits_sha256": "8700e9a12456dce4f744caa329ad2013f44df7f2704e58e245e52aa0e9e9ddc5" + }, + { + "variant": "current", + "tokens": 2048, + "chunk": 512, + "state_sha256": "7f7e23060107d5252391778f46727022f8ee157fa34eb1753c424cac2f3f771f", + "logits_sha256": "8700e9a12456dce4f744caa329ad2013f44df7f2704e58e245e52aa0e9e9ddc5" + }, + { + "variant": "current", + "tokens": 2048, + "chunk": 1024, + "state_sha256": "7f7e23060107d5252391778f46727022f8ee157fa34eb1753c424cac2f3f771f", + "logits_sha256": "8700e9a12456dce4f744caa329ad2013f44df7f2704e58e245e52aa0e9e9ddc5" + }, + { + "variant": "paired_tiles", + "tokens": 2048, + "chunk": 128, + "state_sha256": "a311bb2c6496dc8bbe04a40b8e3d592a3d48c869fdd36b37f6a09dc7b10be651", + "logits_sha256": "7ad8423d5e79a75bc4b72f9a0b32b4f2023272d3a1ec04b597fb37cd5438e964" + }, + { + "variant": "paired_tiles", + "tokens": 2048, + "chunk": 512, + "state_sha256": "a311bb2c6496dc8bbe04a40b8e3d592a3d48c869fdd36b37f6a09dc7b10be651", + "logits_sha256": "7ad8423d5e79a75bc4b72f9a0b32b4f2023272d3a1ec04b597fb37cd5438e964" + }, + { + "variant": "paired_tiles", + "tokens": 2048, + "chunk": 1024, + "state_sha256": "a311bb2c6496dc8bbe04a40b8e3d592a3d48c869fdd36b37f6a09dc7b10be651", + "logits_sha256": "7ad8423d5e79a75bc4b72f9a0b32b4f2023272d3a1ec04b597fb37cd5438e964" + }, + { + "variant": "current", + "tokens": 8192, + "chunk": 128, + "state_sha256": "5524361e72b466c99d75fbf958f1faa0c715920b3456f01532907c47a76cb27b", + "logits_sha256": "19a2f9b7b0794ef2f2be0edb7d16564e24d33f02e0d01972780ae28d09010bf5" + }, + { + "variant": "current", + "tokens": 8192, + "chunk": 512, + "state_sha256": "5524361e72b466c99d75fbf958f1faa0c715920b3456f01532907c47a76cb27b", + "logits_sha256": "19a2f9b7b0794ef2f2be0edb7d16564e24d33f02e0d01972780ae28d09010bf5" + }, + { + "variant": "current", + "tokens": 8192, + "chunk": 1024, + "state_sha256": "5524361e72b466c99d75fbf958f1faa0c715920b3456f01532907c47a76cb27b", + "logits_sha256": "19a2f9b7b0794ef2f2be0edb7d16564e24d33f02e0d01972780ae28d09010bf5" + }, + { + "variant": "paired_tiles", + "tokens": 8192, + "chunk": 128, + "state_sha256": "28e0d790935a448a8d724bd4136b9de38e237afb8dcd545b708e9de5f5b974dd", + "logits_sha256": "305e345e11883986154d8b8dde198985589ac1d064517c14bdd46224c46dd074" + }, + { + "variant": "paired_tiles", + "tokens": 8192, + "chunk": 512, + "state_sha256": "28e0d790935a448a8d724bd4136b9de38e237afb8dcd545b708e9de5f5b974dd", + "logits_sha256": "305e345e11883986154d8b8dde198985589ac1d064517c14bdd46224c46dd074" + }, + { + "variant": "paired_tiles", + "tokens": 8192, + "chunk": 1024, + "state_sha256": "28e0d790935a448a8d724bd4136b9de38e237afb8dcd545b708e9de5f5b974dd", + "logits_sha256": "305e345e11883986154d8b8dde198985589ac1d064517c14bdd46224c46dd074" + } + ], + "state_hash_scope": "Perlayer: complete convolutionstate and validcache.lenpositions ofkeys+values foreachKVhead usinghead*capacity stride; excludesunusedKVcapacity/padding. Final128000BF16logits hashed separately. Equality is within each arithmeticvariant across128/512/1024chunks, notbetween current andpaired variants.", + "diagnostic_artifact_and_source_sha256": { + "zig-out/bin/tf-lfm3-profile": "6545301dec2f8bbcb79193b8560bcae847ecb0f0011e990d2d1162264b14729b", + "zig/tests/lfm3_profile.zig": "1d12a9f4e181635bb4446dd100ac3c28c388500be13a1bed79e843a18b74167c", + "zig/src/core/gpu_profile.zig": "80eb55884ef6d7c9b894f06e3ebfa06a4a485bd3f48895d634ed89d6a638261b", + "zig/src/families/lfm3/forward.zig": "894c9541112518287bc9985db51f7e023b870b234da54acc626e9f2f57533bc9" + }, + "artifact_provenance": "Separate state-enhanced diagnostic6545301d usedfor12proofcases; distinctfrom initialcold-stagef664911c. Qualificationownerreadremoteunchangedartifact/source afterproof andconfirmed no interveningchanges." + }, + "actual_guarded_http": { + "legs": 6, + "all_legs_exited_zero": true, + "primary_requests": 180, + "all_cold_requests_cache0": true, + "all_owned_servers_exited": true, + "lock_acquired": [ + { + "event": "lock_acquired", + "time_ns": 1793202901164008000, + "path": "/tmp/tensorfold-model.lock", + "pid": 20531, + "device": 16777233, + "inode": 2622134 + } + ], + "lock_release": [ + { + "event": "lock_release", + "time_ns": 1793203043859929000, + "pid": 20531 + } + ], + "all_primary_decode_requests_exact256": true, + "cold_output_tokens": 16 + }, + "measured": { + "scope": "M5 TEXT4 current-public batching-only; same remote toolchain/clients/tokens; globalcacheRules128 unchanged; no final performance acceptance", + "engines": { + "batch128": { + "medians": { + "long-code-chat/1": 315.4, + "long-code-chat/4": 861.95, + "long-prose-chat/1": 329.1, + "long-prose-chat/4": 879.4 + }, + "cold_medians": { + "2048": 0.8591289380565286, + "8192": 4.614509062608704 + }, + "legs": [ + { + "tag": "text-4-0-batch128", + "cells": { + "long-code-chat/1": 315.8, + "long-code-chat/4": 861.3, + "long-prose-chat/1": 329.8, + "long-prose-chat/4": 879.0 + }, + "cold": { + "2048": 0.8592449170537293, + "8192": 4.614482833072543 + }, + "cold_hash": { + "2048": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "8192": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349" + } + }, + { + "tag": "text-4-5-batch128", + "cells": { + "long-code-chat/1": 315.0, + "long-code-chat/4": 862.6, + "long-prose-chat/1": 328.4, + "long-prose-chat/4": 879.8 + }, + "cold": { + "2048": 0.8590129590593278, + "8192": 4.614535292144865 + }, + "cold_hash": { + "2048": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "8192": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349" + } + } + ] + }, + "batch512": { + "medians": { + "long-code-chat/1": 315.55, + "long-code-chat/4": 860.45, + "long-prose-chat/1": 329.9, + "long-prose-chat/4": 877.3499999999999 + }, + "cold_medians": { + "2048": 0.8431003752630204, + "8192": 4.529019312467426 + }, + "legs": [ + { + "tag": "text-4-1-batch512", + "cells": { + "long-code-chat/1": 316.0, + "long-code-chat/4": 860.9, + "long-prose-chat/1": 330.3, + "long-prose-chat/4": 878.3 + }, + "cold": { + "2048": 0.8432092503644526, + "8192": 4.529814667068422 + }, + "cold_hash": { + "2048": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "8192": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349" + } + }, + { + "tag": "text-4-4-batch512", + "cells": { + "long-code-chat/1": 315.1, + "long-code-chat/4": 860.0, + "long-prose-chat/1": 329.5, + "long-prose-chat/4": 876.4 + }, + "cold": { + "2048": 0.8429915001615882, + "8192": 4.52822395786643 + }, + "cold_hash": { + "2048": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "8192": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349" + } + } + ] + }, + "batch1024": { + "medians": { + "long-code-chat/1": 314.1, + "long-code-chat/4": 859.9, + "long-prose-chat/1": 327.95, + "long-prose-chat/4": 877.55 + }, + "cold_medians": { + "2048": 0.8394205416552722, + "8192": 4.506114687537774 + }, + "legs": [ + { + "tag": "text-4-2-batch1024", + "cells": { + "long-code-chat/1": 314.2, + "long-code-chat/4": 860.0, + "long-prose-chat/1": 327.9, + "long-prose-chat/4": 877.2 + }, + "cold": { + "2048": 0.8395577501505613, + "8192": 4.506872625090182 + }, + "cold_hash": { + "2048": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "8192": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349" + } + }, + { + "tag": "text-4-3-batch1024", + "cells": { + "long-code-chat/1": 314.0, + "long-code-chat/4": 859.8, + "long-prose-chat/1": 328.0, + "long-prose-chat/4": 877.9 + }, + "cold": { + "2048": 0.8392833331599832, + "8192": 4.505356749985367 + }, + "cold_hash": { + "2048": "a9f963e9ea3381adb9633cbe50e9c3e84ba88e0f646c6280583320a8097c1da9", + "8192": "e2476773ac8616580547ee0cb0ea44121f776b7988e7eba184743b0d51441349" + } + } + ] + } + }, + "cold_response_hashes_equal_across_chunks": true, + "cross_chunk_full_256_token_hashes": { + "long-code-chat": [ + "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7" + ], + "long-prose-chat": [ + "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a" + ] + }, + "all_cross_chunk_decode_hashes_equal_per_prompt": true, + "primary_measured_requests": 180 + }, + "conclusion": "1024 reduces median8kTTFT2.35percent with flatdecode; this does not resolve the measured standard-server performance gap.", + "performance_accepted": false, + "raw_evidence_sha256": { + "batch-proof-cpu-overlap/0-current-2048-128.json": "d7c086e19e408df07d1328804f7cedf1afcf977b69604ce7920c8adbf8227730", + "batch-proof-cpu-overlap/0-current-2048-128.log": "3abf7120a89d3ff17e97e5d25061c4199236419aba394e4cada6f084877cc954", + "batch-proof-cpu-overlap/1-current-2048-512.json": "ac686900ba9848b537684ac8b821f902bcff39916c9a2233178320479b6963d3", + "batch-proof-cpu-overlap/1-current-2048-512.log": "027e330fce708cd3666505e90436b8b18d6d7a977b16d760bc3e34194ec8f325", + "batch-proof-cpu-overlap/10-paired-8192-512.json": "e07ca97c54659b36bb6c241ba25d5f10903dd12a01cee159ce19a1e27aaa5a8a", + "batch-proof-cpu-overlap/10-paired-8192-512.log": "e1b2a45a79d96cdab9d8cc6f875d25100ac33ba777cf7639f6b2ee460514311c", + "batch-proof-cpu-overlap/11-paired-8192-1024.json": "5642e80e5851847bc5feb4e57985ce9ae34ccd5c7bad35fd647f038d5febbfa0", + "batch-proof-cpu-overlap/11-paired-8192-1024.log": "693aaafbf1ce09f8c1ad859ff46b42f7bb9e4e74d8c233de717a22a2675e2e18", + "batch-proof-cpu-overlap/2-current-2048-1024.json": "df587499c06db59e9bfcc7f54391f707f82a2fafa0b108ca947a9c45fc9fa9f8", + "batch-proof-cpu-overlap/2-current-2048-1024.log": "a6e60d7b73270eec97e8d3395c57663cf09d00fd16cead0afb91a331d96e5492", + "batch-proof-cpu-overlap/3-paired-2048-128.json": "bf07642f6511d0209999dbde294bdb92553ad8183ba63f9369047999630cc5e3", + "batch-proof-cpu-overlap/3-paired-2048-128.log": "3abf7120a89d3ff17e97e5d25061c4199236419aba394e4cada6f084877cc954", + "batch-proof-cpu-overlap/4-paired-2048-512.json": "17ff3800e2009858f67ea68f040a22fef1f79b0cbd64a0c3b036b6f0e8cf7014", + "batch-proof-cpu-overlap/4-paired-2048-512.log": "027e330fce708cd3666505e90436b8b18d6d7a977b16d760bc3e34194ec8f325", + "batch-proof-cpu-overlap/5-paired-2048-1024.json": "b49529ef675762a299338d2595824afd6f6ff6804868888422a58b968f4cc9eb", + "batch-proof-cpu-overlap/5-paired-2048-1024.log": "a6e60d7b73270eec97e8d3395c57663cf09d00fd16cead0afb91a331d96e5492", + "batch-proof-cpu-overlap/6-current-8192-128.json": "a75245e722d5849f48b6762d60bec9dd4d5ec642baeb5ed5f48276e75e9e96fc", + "batch-proof-cpu-overlap/6-current-8192-128.log": "5cf360bde5c1a9e0ff68b1cbc385982da901a621793fe54b8ef9fac5763c83ef", + "batch-proof-cpu-overlap/7-current-8192-512.json": "0e423b24b7497aa7b37ccd81e40229a240a28cdf8a36b89088184d588a40e062", + "batch-proof-cpu-overlap/7-current-8192-512.log": "e1b2a45a79d96cdab9d8cc6f875d25100ac33ba777cf7639f6b2ee460514311c", + "batch-proof-cpu-overlap/8-current-8192-1024.json": "eb72b25ac6460aae682d36d395a6e077775db5f104d65209a997ffef819cf85a", + "batch-proof-cpu-overlap/8-current-8192-1024.log": "693aaafbf1ce09f8c1ad859ff46b42f7bb9e4e74d8c233de717a22a2675e2e18", + "batch-proof-cpu-overlap/9-paired-8192-128.json": "9be85bc2d4a7380aa4e4e5bfd2e824bc6a59438d86eddf886e9b35d860c14f81", + "batch-proof-cpu-overlap/9-paired-8192-128.log": "5cf360bde5c1a9e0ff68b1cbc385982da901a621793fe54b8ef9fac5763c83ef", + "batch-proof-cpu-overlap/events.json": "908cb91625c9b6cc47be33695f68b4e885285ae3e9abea2cc4896ba317495e64", + "batch-proof-cpu-overlap/records.json": "7dbfdaf498e0351d843aad4dc24ce5aeb2cb14e8c59b4ecc0d37007ba703d3f6", + "batch-http-results/events.json": "d0c7ee00352eb24cf4e9f62d517103104b89b310414a1803fc30e6979bf3c813", + "batch-http-results/manifest.json": "1621296c8c2c5dd4418ce76df95d8852426d51934c85ebbc596d8600e8596714", + "batch-http-results/records.json": "6720c00b383b1be051ac5b94c94bb75d1b911ce062459c7cc5fe217c106c6fe8", + "batch-http-results/summary.json": "1fd8d85dde73f53b7df030eec3cadf8f645221fd8f70fca24fac54f21ae4c5e1", + "batch-http-results/text-4-0-batch128.after.health": "aafbd1bfafda3d655953e1fbf374a4b538ac6beb86745507d8f0e52ac7bc8da5", + "batch-http-results/text-4-0-batch128.after.metrics": "8186f92fe0b3a978ffb003033065fae6afdc795fa54cdc8cede57b9766d1031d", + "batch-http-results/text-4-0-batch128.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-0-batch128.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "batch-http-results/text-4-0-batch128.before.metrics": "71f2b5590d688aa2e3eb65bfbb767409db9ded4db139eac8b46ef95a0671a604", + "batch-http-results/text-4-0-batch128.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-0-batch128.cold-2048.client.log": "8cca6d4bfaa730306250057111c9d4f7b5c2698d0a9079b0a1a49c5dc079efce", + "batch-http-results/text-4-0-batch128.cold-2048.json": "0833705cb3eebd04c104a4f99b9605e1cbcecb244e53a1a2d582be392975cc78", + "batch-http-results/text-4-0-batch128.cold-8192.client.log": "8be2f7473963c3cfde53c441e2c4fb0a49037c64a8e97d1fe115eae847a6c331", + "batch-http-results/text-4-0-batch128.cold-8192.json": "2f11c23b2183b49dcd9160bfaf63a9030ac219ee8b92e47a4dfa4553685943af", + "batch-http-results/text-4-0-batch128.concurrent.client.log": "807bc346faaf103ddffcc7a763639467fce0c5f52692a0fbe7e4c2a0f021ebef", + "batch-http-results/text-4-0-batch128.concurrent.json": "2dda4a19c1b7e72931e842531ad7dfbfd48cbb1cd9ef15d927d709b8dd04552c", + "batch-http-results/text-4-0-batch128.server.log": "07efb3e16cd0a2c3fac73935e2db2990f06126ca3428101109d19a700ae066b9", + "batch-http-results/text-4-1-batch512.after.health": "490b87e88a6d848b908a53902677ceb3f29e5e9806fe03965664b819148e0c9c", + "batch-http-results/text-4-1-batch512.after.metrics": "5c119b56ce47469448efcd47cf2bc799e85f6dbe16565dc6847ae4241cca850e", + "batch-http-results/text-4-1-batch512.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-1-batch512.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "batch-http-results/text-4-1-batch512.before.metrics": "db6147183ab871ff468ddb40575e4ec477b405be22791d2c03cefcfc329c0655", + "batch-http-results/text-4-1-batch512.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-1-batch512.cold-2048.client.log": "785973d50b59b117f1fc6fc1926bb4b281087f59404c95cb14e1e64e69ebf224", + "batch-http-results/text-4-1-batch512.cold-2048.json": "07d5a9cc103cac3c15336534edb828f4f3d254c69e1b700df45d4249814bb017", + "batch-http-results/text-4-1-batch512.cold-8192.client.log": "7386f9a64b191eded0cfb409840795096d5ed1d99c3bd1ccda096047aa24d727", + "batch-http-results/text-4-1-batch512.cold-8192.json": "22f7ae4d64d6ecf7e12703c0ac5f59bb4e2cc1e72fb9e16633544c5973fd86c4", + "batch-http-results/text-4-1-batch512.concurrent.client.log": "45785f02d0734998e11c9122e872b814108b9ea54e7607a36acb825a3be87fda", + "batch-http-results/text-4-1-batch512.concurrent.json": "34175439bdbe5e8cc015b5ccdd6535a6b345b35ba6f63fdef5aa7f0992dda0d6", + "batch-http-results/text-4-1-batch512.server.log": "ebb4a315385b2dabda8679787cf70d0f0448c3bd87aebc78119649dca5b8c7f9", + "batch-http-results/text-4-2-batch1024.after.health": "dbb1feaf616e1a00d04401c87f0109647587ea10085e790160b396a55684aacb", + "batch-http-results/text-4-2-batch1024.after.metrics": "a4c1965c89d265bf7a92b2304f68638664dad15d6eb1284ff020aff5dfdd3537", + "batch-http-results/text-4-2-batch1024.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-2-batch1024.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "batch-http-results/text-4-2-batch1024.before.metrics": "533736c30e067b86991f68140fbc3e2c46e2a52164c690bb14b8700445131b3d", + "batch-http-results/text-4-2-batch1024.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-2-batch1024.cold-2048.client.log": "0f9b810b58f749e2f6e85969a04b0d445676ac11fa1869bdbe40b6e69d7da6c1", + "batch-http-results/text-4-2-batch1024.cold-2048.json": "67dada418b4ab98f232f61dddcdcce81a8ef1828b286725bc4b089b1a037f66b", + "batch-http-results/text-4-2-batch1024.cold-8192.client.log": "45d0a2919e3318305e647e6f5e0ecac838d19f1f52efe8899ba9a0ec423c39d3", + "batch-http-results/text-4-2-batch1024.cold-8192.json": "c15cfd990c2ddd82da49a8b786b3504855f5ef95b4f52b8d350a0b18d39fe5c7", + "batch-http-results/text-4-2-batch1024.concurrent.client.log": "a719a41f39e5a79fbd064707d72909c83054ab67773edbea1b1147d26fef360e", + "batch-http-results/text-4-2-batch1024.concurrent.json": "de5d7b38b8f8a9572ad3e29a97b54c661daecf1e84fc236adcae41a1ba5de032", + "batch-http-results/text-4-2-batch1024.server.log": "f2390a73ff9affd4977f6d086caa29ffaed789ab7426b43f44ad403f4a0ec27f", + "batch-http-results/text-4-3-batch1024.after.health": "a47e0156c35f771a328162673c73a84f98cdb4fab472190dce8838634b793f06", + "batch-http-results/text-4-3-batch1024.after.metrics": "c872474a26661c0934a8d3a0bcfa2e9b78a631cf88deeba16bece74e7d186094", + "batch-http-results/text-4-3-batch1024.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-3-batch1024.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "batch-http-results/text-4-3-batch1024.before.metrics": "af3c161b85c32f5f64487695e2a5206fd927bf093fe8be10b127ee435dffd592", + "batch-http-results/text-4-3-batch1024.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-3-batch1024.cold-2048.client.log": "85d2effaa59ed328ea3a6294ccc6517ebfd2685241e8a9c563c7e5cef88fc1be", + "batch-http-results/text-4-3-batch1024.cold-2048.json": "51e62bb54cd232cefb58d055bb9935e7fac673edd973244abd0f5a094a72de86", + "batch-http-results/text-4-3-batch1024.cold-8192.client.log": "9b5dad818b31007586ce4de023acb638e8b13c07fb9c4738be7c5b8bb569960f", + "batch-http-results/text-4-3-batch1024.cold-8192.json": "1819ca126b1f917928c296e0b013d52ddb8cef47ebc84b555373409d50b7a0c3", + "batch-http-results/text-4-3-batch1024.concurrent.client.log": "08265839e8715440942bbab88978e8e3291aa68cf9ed9f63f0595573336fadc2", + "batch-http-results/text-4-3-batch1024.concurrent.json": "40cb038f367ab1e5d5cbcbb3e4e267d168d37101044288982cf13b30c9e5b83a", + "batch-http-results/text-4-3-batch1024.server.log": "46864e805da2d76e6f4dc429ce886026305ff3ab147662a02d1f0df05c24e5ef", + "batch-http-results/text-4-4-batch512.after.health": "7089918ee646a3c7c0d15100f552a3636bb7eda622c10390507b0b6ff60c9063", + "batch-http-results/text-4-4-batch512.after.metrics": "13afef5db57a45083269b5261d370af70450af121f8fbf05b76f87203c6f1b8a", + "batch-http-results/text-4-4-batch512.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-4-batch512.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "batch-http-results/text-4-4-batch512.before.metrics": "9b62fcd32fcebfe444483006d14e938fdf7a0225a6b14ffeb07c1f11c98e612d", + "batch-http-results/text-4-4-batch512.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-4-batch512.cold-2048.client.log": "1b421e1e4f2d2b7608d8088f8fde3f9f0e5b3a72b2d66dba57ccd24906ff04ec", + "batch-http-results/text-4-4-batch512.cold-2048.json": "6aeea0380bd51b9396afd86bf2e0f37590d96e32a2b2e1149250d64ddccb1471", + "batch-http-results/text-4-4-batch512.cold-8192.client.log": "813b2e43ad7a54234963032faaff19191bd5a617198b897ea9f3d38f6e8e12f5", + "batch-http-results/text-4-4-batch512.cold-8192.json": "378cf0bb89e218a5f66775632391c8eca1503260195fec877cc15f2949ce0c59", + "batch-http-results/text-4-4-batch512.concurrent.client.log": "5ef46ac876e97622c5db583884cbddad7af9287cf52e506941fb71639a3c411a", + "batch-http-results/text-4-4-batch512.concurrent.json": "5f936794654c24b12f24409f94fc797338e5160080a4bfee719f2bffd9e8a725", + "batch-http-results/text-4-4-batch512.server.log": "4e4bfa2837c027a11ec79429f9e4dc84ca7d4ece2e3b4100397b75283ea9d837", + "batch-http-results/text-4-5-batch128.after.health": "d5a1b186460fef2802d4f6632c6c78486fdc920617b50b2966a40c3e33e30938", + "batch-http-results/text-4-5-batch128.after.metrics": "cbe61937c7b5d3c06fd6784def48ff320aa113c3681e3139cf4406b735deed3a", + "batch-http-results/text-4-5-batch128.after.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-5-batch128.before.health": "e41f1fa5fa63b41398632c5ab9622e881079971c2bec31b6d54da709ab256599", + "batch-http-results/text-4-5-batch128.before.metrics": "a13c9b31b693edb8615b4906cb566a2321c1367926ab6679144d3d8eba6ad5b7", + "batch-http-results/text-4-5-batch128.before.v1-settings.unavailable": "6e3c5ec0e30e80a389805137574a443680f3f648a64e35bc4bff31e642f53da5", + "batch-http-results/text-4-5-batch128.cold-2048.client.log": "a74aa6c77fc854ccb492ee137bcf7fadb9125385c0923269be9b67139d40480f", + "batch-http-results/text-4-5-batch128.cold-2048.json": "d1861a0ecdaaecc7ba59cd0faaf2f71746d6ab073035634f5f1840364fa25b96", + "batch-http-results/text-4-5-batch128.cold-8192.client.log": "e68eba7024c7dd71d2a71f4720c19f12535ca500b1292ae86f56eb04038c38f1", + "batch-http-results/text-4-5-batch128.cold-8192.json": "34a9115628fde19cfadab69b1b6f83cde92670a54b01a2c8fe3a8848f773e331", + "batch-http-results/text-4-5-batch128.concurrent.client.log": "6b72967dd74c86a261e184d8005e250a10a49891c2f410400c36d7bef39c2483", + "batch-http-results/text-4-5-batch128.concurrent.json": "e3af1986b198812ffdcada6fca074b2eceace600c2740c4cb891764fea15d69a", + "batch-http-results/text-4-5-batch128.server.log": "01172e5369920ed37fff9b52d38241137d2f42228486c6aeccad23155be9c3f0" + } +} diff --git a/docs/receipts/lfm3/model-fp64-comparison.json b/docs/receipts/lfm3/model-fp64-comparison.json new file mode 100644 index 0000000000..c2170d5167 --- /dev/null +++ b/docs/receipts/lfm3/model-fp64-comparison.json @@ -0,0 +1,559 @@ +{ + "cells": 17, + "rows": 18, + "native_in_oracle_maximum": 9, + "mlx_in_oracle_maximum": 16, + "both_in_maximum": 8, + "neither_in_maximum": 1, + "accepted_threshold": null, + "rows_detail": [ + { + "row": 82, + "native_top": 2292, + "mlx_top": 3599, + "oracle_maximum_set": [ + 2292, + 3599 + ], + "oracle_margin": 0.0, + "native_in_oracle_maximum_set": true, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.0, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 82, + "reference_top": 3599, + "native_top": 2292, + "reference_top_set": [ + 3599 + ], + "native_top_set": [ + 2292 + ], + "top_sets_overlap": false, + "reference_margin": 0.125, + "native_margin": 0.5, + "reference_native_contender_gap": 0.125, + "native_reference_contender_gap": 0.5 + }, + "cell": "1.7B-6-prose-257" + }, + { + "row": 82, + "native_top": 2292, + "mlx_top": 3599, + "oracle_maximum_set": [ + 2292, + 3599 + ], + "oracle_margin": 0.0, + "native_in_oracle_maximum_set": true, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.0, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 82, + "reference_top": 3599, + "native_top": 2292, + "reference_top_set": [ + 3599 + ], + "native_top_set": [ + 2292 + ], + "top_sets_overlap": false, + "reference_margin": 0.125, + "native_margin": 0.5, + "reference_native_contender_gap": 0.125, + "native_reference_contender_gap": 0.5 + }, + "cell": "1.7B-6-prose-513" + }, + { + "row": 450, + "native_top": 22497, + "mlx_top": 360, + "oracle_maximum_set": [ + 22497 + ], + "oracle_margin": 0.125, + "native_in_oracle_maximum_set": true, + "mlx_in_oracle_maximum_set": false, + "oracle_gap_to_native_top": 0.0, + "oracle_gap_to_mlx_top": 0.125, + "previous_evidence": { + "row": 450, + "reference_top": 360, + "native_top": 22497, + "reference_top_set": [ + 360 + ], + "native_top_set": [ + 22497 + ], + "top_sets_overlap": false, + "reference_margin": 0.125, + "native_margin": 0.125, + "reference_native_contender_gap": 0.125, + "native_reference_contender_gap": 0.125 + }, + "cell": "1.7B-8-code-513" + }, + { + "row": 264, + "native_top": 207, + "mlx_top": 33237, + "oracle_maximum_set": [ + 2993 + ], + "oracle_margin": 0.375, + "native_in_oracle_maximum_set": false, + "mlx_in_oracle_maximum_set": false, + "oracle_gap_to_native_top": 0.375, + "oracle_gap_to_mlx_top": 0.5625, + "previous_evidence": { + "row": 264, + "reference_top": 33237, + "native_top": 207, + "reference_top_set": [ + 33237 + ], + "native_top_set": [ + 207 + ], + "top_sets_overlap": false, + "reference_margin": 0.0625, + "native_margin": 0.25, + "reference_native_contender_gap": 0.0625, + "native_reference_contender_gap": 0.25 + }, + "cell": "VL-2B-4-code-513" + }, + { + "row": 17, + "native_top": 296, + "mlx_top": 405, + "oracle_maximum_set": [ + 405 + ], + "oracle_margin": 0.125, + "native_in_oracle_maximum_set": false, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.125, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 17, + "reference_top": 405, + "native_top": 296, + "reference_top_set": [ + 405 + ], + "native_top_set": [ + 296 + ], + "top_sets_overlap": false, + "reference_margin": 0.0625, + "native_margin": 0.125, + "reference_native_contender_gap": 0.0625, + "native_reference_contender_gap": 0.1875 + }, + "cell": "VL-2B-4-reasoning-257" + }, + { + "row": 17, + "native_top": 296, + "mlx_top": 405, + "oracle_maximum_set": [ + 405 + ], + "oracle_margin": 0.125, + "native_in_oracle_maximum_set": false, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.125, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 17, + "reference_top": 405, + "native_top": 296, + "reference_top_set": [ + 405 + ], + "native_top_set": [ + 296 + ], + "top_sets_overlap": false, + "reference_margin": 0.0625, + "native_margin": 0.125, + "reference_native_contender_gap": 0.0625, + "native_reference_contender_gap": 0.1875 + }, + "cell": "VL-2B-4-reasoning-513" + }, + { + "row": 22, + "native_top": 16033, + "mlx_top": 91, + "oracle_maximum_set": [ + 91 + ], + "oracle_margin": 0.1875, + "native_in_oracle_maximum_set": false, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.1875, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 22, + "reference_top": 91, + "native_top": 16033, + "reference_top_set": [ + 91 + ], + "native_top_set": [ + 16033 + ], + "top_sets_overlap": false, + "reference_margin": 0.125, + "native_margin": 0.125, + "reference_native_contender_gap": 0.125, + "native_reference_contender_gap": 0.125 + }, + "cell": "VL-2B-6-code-257" + }, + { + "row": 83, + "native_top": 3570, + "mlx_top": 496, + "oracle_maximum_set": [ + 496, + 3570 + ], + "oracle_margin": 0.0, + "native_in_oracle_maximum_set": true, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.0, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 83, + "reference_top": 496, + "native_top": 3570, + "reference_top_set": [ + 496 + ], + "native_top_set": [ + 3570 + ], + "top_sets_overlap": false, + "reference_margin": 0.1875, + "native_margin": 0.125, + "reference_native_contender_gap": 0.1875, + "native_reference_contender_gap": 0.1875 + }, + "cell": "VL-2B-6-code-257" + }, + { + "row": 83, + "native_top": 3570, + "mlx_top": 496, + "oracle_maximum_set": [ + 496, + 3570 + ], + "oracle_margin": 0.0, + "native_in_oracle_maximum_set": true, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.0, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 83, + "reference_top": 496, + "native_top": 3570, + "reference_top_set": [ + 496 + ], + "native_top_set": [ + 3570 + ], + "top_sets_overlap": false, + "reference_margin": 0.1875, + "native_margin": 0.125, + "reference_native_contender_gap": 0.1875, + "native_reference_contender_gap": 0.1875 + }, + "cell": "VL-2B-6-code-513" + }, + { + "row": 22, + "native_top": 16033, + "mlx_top": 91, + "oracle_maximum_set": [ + 91 + ], + "oracle_margin": 0.1875, + "native_in_oracle_maximum_set": false, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.1875, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 22, + "reference_top": 91, + "native_top": 16033, + "reference_top_set": [ + 91 + ], + "native_top_set": [ + 16033 + ], + "top_sets_overlap": false, + "reference_margin": 0.0625, + "native_margin": 0.125, + "reference_native_contender_gap": 0.0625, + "native_reference_contender_gap": 0.125 + }, + "cell": "VL-2B-6-code-64" + }, + { + "row": 38, + "native_top": 22, + "mlx_top": 20, + "oracle_maximum_set": [ + 20, + 22 + ], + "oracle_margin": 0.0, + "native_in_oracle_maximum_set": true, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.0, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 38, + "reference_top": 20, + "native_top": 22, + "reference_top_set": [ + 20 + ], + "native_top_set": [ + 22 + ], + "top_sets_overlap": false, + "reference_margin": 0.125, + "native_margin": 0.125, + "reference_native_contender_gap": 0.125, + "native_reference_contender_gap": 0.125 + }, + "cell": "VL-2B-6-prose-257" + }, + { + "row": 38, + "native_top": 22, + "mlx_top": 20, + "oracle_maximum_set": [ + 20, + 22 + ], + "oracle_margin": 0.0, + "native_in_oracle_maximum_set": true, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.0, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 38, + "reference_top": 20, + "native_top": 22, + "reference_top_set": [ + 20 + ], + "native_top_set": [ + 22 + ], + "top_sets_overlap": false, + "reference_margin": 0.125, + "native_margin": 0.125, + "reference_native_contender_gap": 0.125, + "native_reference_contender_gap": 0.125 + }, + "cell": "VL-2B-6-prose-513" + }, + { + "row": 17, + "native_top": 3547, + "mlx_top": 3983, + "oracle_maximum_set": [ + 3983 + ], + "oracle_margin": 0.0625, + "native_in_oracle_maximum_set": false, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.0625, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 17, + "reference_top": 3983, + "native_top": 3547, + "reference_top_set": [ + 3983 + ], + "native_top_set": [ + 3547 + ], + "top_sets_overlap": false, + "reference_margin": 0.0625, + "native_margin": 0.0625, + "reference_native_contender_gap": 0.0625, + "native_reference_contender_gap": 0.0625 + }, + "cell": "VL-2B-6-prose-64" + }, + { + "row": 24, + "native_top": 27, + "mlx_top": 1325, + "oracle_maximum_set": [ + 1325 + ], + "oracle_margin": 0.15625, + "native_in_oracle_maximum_set": false, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.15625, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 24, + "reference_top": 1325, + "native_top": 27, + "reference_top_set": [ + 1325 + ], + "native_top_set": [ + 27 + ], + "top_sets_overlap": false, + "reference_margin": 0.0625, + "native_margin": 0.09375, + "reference_native_contender_gap": 0.0625, + "native_reference_contender_gap": 0.09375 + }, + "cell": "VL-2B-6-reasoning-257" + }, + { + "row": 24, + "native_top": 27, + "mlx_top": 1325, + "oracle_maximum_set": [ + 1325 + ], + "oracle_margin": 0.15625, + "native_in_oracle_maximum_set": false, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.15625, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 24, + "reference_top": 1325, + "native_top": 27, + "reference_top_set": [ + 1325 + ], + "native_top_set": [ + 27 + ], + "top_sets_overlap": false, + "reference_margin": 0.125, + "native_margin": 0.09375, + "reference_native_contender_gap": 0.125, + "native_reference_contender_gap": 0.09375 + }, + "cell": "VL-2B-6-reasoning-513" + }, + { + "row": 24, + "native_top": 27, + "mlx_top": 1325, + "oracle_maximum_set": [ + 1325 + ], + "oracle_margin": 0.15625, + "native_in_oracle_maximum_set": false, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.15625, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 24, + "reference_top": 1325, + "native_top": 27, + "reference_top_set": [ + 1325 + ], + "native_top_set": [ + 27 + ], + "top_sets_overlap": false, + "reference_margin": 0.25, + "native_margin": 0.09375, + "reference_native_contender_gap": 0.25, + "native_reference_contender_gap": 0.09375 + }, + "cell": "VL-2B-6-reasoning-64" + }, + { + "row": 12, + "native_top": 20, + "mlx_top": 296, + "oracle_maximum_set": [ + 20, + 296 + ], + "oracle_margin": 0.0, + "native_in_oracle_maximum_set": true, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.0, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 12, + "reference_top": 296, + "native_top": 20, + "reference_top_set": [ + 296 + ], + "native_top_set": [ + 20 + ], + "top_sets_overlap": false, + "reference_margin": 0.0625, + "native_margin": 0.0625, + "reference_native_contender_gap": 0.0625, + "native_reference_contender_gap": 0.0625 + }, + "cell": "VL-2B-8-prose-64" + }, + { + "row": 16, + "native_top": 20, + "mlx_top": 22, + "oracle_maximum_set": [ + 20, + 22 + ], + "oracle_margin": 0.0, + "native_in_oracle_maximum_set": true, + "mlx_in_oracle_maximum_set": true, + "oracle_gap_to_native_top": 0.0, + "oracle_gap_to_mlx_top": 0.0, + "previous_evidence": { + "row": 16, + "reference_top": 22, + "native_top": 20, + "reference_top_set": [ + 22 + ], + "native_top_set": [ + 20 + ], + "top_sets_overlap": false, + "reference_margin": 0.0625, + "native_margin": 0.0625, + "reference_native_contender_gap": 0.0625, + "native_reference_contender_gap": 0.0625 + }, + "cell": "VL-2B-8-reasoning-513" + } + ] +} diff --git a/docs/receipts/lfm3/postnorm-fusion.json b/docs/receipts/lfm3/postnorm-fusion.json new file mode 100644 index 0000000000..a6697c3959 --- /dev/null +++ b/docs/receipts/lfm3/postnorm-fusion.json @@ -0,0 +1,383 @@ +{ + "scope": "Qualified private exact post-add plus next RMSNorm fusion; public application requires rebuilding and checking the applied source. Performance measured only for the two 4-bit checkpoints.", + "base_commit": "6b318cedb9d9520ee53ee4a56abeb5d6ced5d461", + "algorithm": "Fuse 48 adjacent dispatch pairs per forward; retain both unchanged 256-thread FP32 reduction trees, all BF16 round sites and the residual h store. No carry crosses a forward call. Norm-class dispatches decrease from 97 to 49.", + "source_sha256": { + "zig/kernels/metal/lfm3/ops.metal": "1659bb9185d966dd30276a6668465d25c06753a03c4cb38a02c3684514a657ae", + "zig/src/families/lfm3/forward.zig": "1ab4e29b47056c3232d3e8278c40c6b7dcf295d52d1f172e71ea47d95e92ff0f", + "zig/src/families/lfm3/kernels.zig": "7f04b485a2e9d1dc5ad6d2c7e49a4278ad61128fb946daee878a665e7ba21fed", + "zig/tests/contract/metal_lfm3_postnorm.zig": "8c362ca20a90fe093f1f08d65ace30affabe63879456ba0394ba84532d688594", + "zig/tests/contract/metal_main.zig": "3eff9d0bb4cfdbd0c990ed4201a094bcf91107e0e2ad5c4792551eda85db6922", + "zig/tests/contract/metal_limits.zig": "5d74368ae9b00d65373421719423ba98b6627d02645909d114e418ab9cc2a0e9", + "zig/src/core/contract/lfm3_records.zig": "33cf7203d5e371a016cceecb165923355a72dbcc4eadd1a3ceb6a92dbad0bd54" + }, + "native_tested_source_sha256": { + "zig/kernels/metal/lfm3/ops.metal": "1659bb9185d966dd30276a6668465d25c06753a03c4cb38a02c3684514a657ae", + "zig/src/families/lfm3/forward.zig": "1ab4e29b47056c3232d3e8278c40c6b7dcf295d52d1f172e71ea47d95e92ff0f", + "zig/src/families/lfm3/kernels.zig": "b5f5406605f665cd6ab3bacbeca4ccf53c39245d13e827ab0819247a981b6dfa" + }, + "source_provenance": "After native binaries were frozen, kernels.zig changed only Zig formatting. Later changes affect contract metadata/reference/simulation only, not runtime arithmetic.", + "artifact_sha256": { + "zig-out/native/bin/tensorfold-native": "a20ea31aa466b7e20a85022a6932ad069ae4f9100d899d9a70743daef9132bae", + "zig-out/bin/tf-lfm3-forward": "4e350b7d02a793d375ba72db48b468b14e1255bfd47e51f9913429befaeea186", + "zig-out/bin/tf-lfm3-exact": "bd899ef674c8668fb8dc7491f45540d4435c070bb4f5bc500af4573f21312286", + "zig-out/bin/tf-kernel-contract": "c29de86253f66822af91b73f753b65247e42a33c41aeaa171e79223b1607542c" + }, + "hosts": [ + { + "chip": "Apple M4 Pro", + "macos": "26.5.1", + "memory_gib": 48 + }, + { + "chip": "Apple M5 Ultra", + "macos": "27.0", + "memory_gib": 256 + } + ], + "primitive": { + "hosts": [ + "M4 Pro", + "M5 Ultra" + ], + "cases_per_host": 320, + "bytes_equal_per_host": 124354560, + "rows": [ + 1, + 2, + 3, + 4, + 5, + 7, + 8, + 9, + 15, + 16, + 17, + 31, + 32, + 33, + 63, + 64, + 65, + 127, + 128, + 129 + ], + "distributions": [ + "random finite", + "signed zero", + "BF16 subnormal", + "maximum finite", + "NaN", + "positive infinity", + "negative infinity", + "random raw BF16" + ], + "outputs": [ + "residual BF16", + "next normalized BF16", + "next normalized pre-store FP32" + ], + "guards": true, + "independent_post_and_next_gain_inputs": true, + "embedded_source_sha256": "e09a71920a04f1e6f864ff9c7a71db3b57f3b23b6fde4efdeb31a372374c240c", + "driver_sha256": "fc67f80fae27155c327c8b161c9585202b7358bee201b55867bf158525a5545e", + "binary_sha256": "65764baa7acf6b5cb76e8400f222a7e07d06169c84c9ee442ddf2de8e7380755" + }, + "canonical_model_logits": { + "hosts": [ + "M4 Pro", + "M5 Ultra" + ], + "quantizations_per_host": 6, + "rows_per_checkpoint": 257, + "bytes_equal_per_host": 394752000 + }, + "trace": { + "files": 620, + "bytes_equal": 68474208, + "rows": 33, + "chunk": 32 + }, + "image_conditioned": { + "family": "VL8", + "rows": 90, + "bytes_equal": 23040000, + "sha256": "db730509b6d7dcb3d6d134b59b6434dfb43ebc85cdd3fe351bc8cc4148b47642" + }, + "state": { + "host": "M4 Pro", + "models": 6, + "passed": 408, + "failed": 0, + "skipped": 0, + "per_checkpoint_passed": 68, + "binary_sha256": "bd899ef674c8668fb8dc7491f45540d4435c070bb4f5bc500af4573f21312286", + "log_sha256": { + "LFM3-1.7B-MLX-4bit": "3f460abee50d57fe6fbde0290b441415f9a7e748f6a79e69088d77ca87cd8ec2", + "LFM3-1.7B-MLX-6bit": "16075f3068f51a7baa8edff1c73a504275b3798d917d33a3596f6bff85baf000", + "LFM3-1.7B-MLX-8bit": "db4169b0e9803ef90f230c3896ba51f0a664e7f29a3ae65af8cd321319c77a92", + "LFM3-VL-2B-MLX-4bit": "005371e44f10619eaa1aeac4b1438334bf34cdd70abcf603a325ebc0a2cc72b8", + "LFM3-VL-2B-MLX-6bit": "5723cdb4ceebcb5ab9668d77d2a8f4d7a21acbf4be99c104b3359d734ba7a75c", + "LFM3-VL-2B-MLX-8bit": "ffca558c50d192a2e296ee82d66a8b527dc1cb3279f080b37f7c1ef6cc25b978" + } + }, + "serving": { + "host": "M5 Ultra", + "models": 6, + "passed": 303, + "failed": 0, + "skipped": 0, + "modes": [ + "greedy shared versus solo", + "seeded sampling", + "cancellation", + "vision inputs" + ], + "receipt_sha256": { + "LFM3-1.7B-MLX-4bit": "d0ba2a8adc090d2e40259456e65e04f60fe2157f3221369b3b961336ff17f408", + "LFM3-1.7B-MLX-6bit": "d9634eeb767ba195531d35e1694dce4df7bbcc38b6782a6825ce456fc96c6a4c", + "LFM3-1.7B-MLX-8bit": "6e1108d7f5158fc27c8560a06a595614bd1d3bf746005c24b9a36bf2ec61abce", + "LFM3-VL-2B-MLX-4bit": "7337b3300a191858cef3858fc813739e99fbe0704f803b64f911783ab04a2642", + "LFM3-VL-2B-MLX-6bit": "fbcc9fec889ff54adcd61e2371d20bf8682a663829bc76b0194c08fcf456d33d", + "LFM3-VL-2B-MLX-8bit": "e9ca9565030968665abb8585c922d768494b095862b2eecf4680c174f4f19640" + } + }, + "contract": { + "hosts": [ + "M4 Pro", + "M5 Ultra" + ], + "kernels": 1, + "gates_1_4_pass": true, + "rounding_controls_red": 1, + "normal_launch_records": 355, + "limit_problems": 0, + "threads": 256, + "static_threadgroup_bytes": 144, + "declared_intermediate_bf16_rounds": 4, + "round_sites": [ + "first normalization before gain", + "first normalization after gain", + "residual addition", + "second normalization before gain" + ], + "prior_stricter_mids2_probe": { + "host": "Apple M4 Pro", + "result": "all four gates pass and removed-rounding control red", + "source_sha256": "27e59b782478d210f80ee5fbee52fd7f156ce94b5b69922dd555dfe74232362a", + "binary_sha256": "5b62de7efd7280a78b7ddeee98dae58ae26015b9ef4dcff4bd6b78d23309571d", + "log_sha256": "b68f40af159eac40419ca1a73878c400e78f24210334dd1cf395ea963617594f" + }, + "metadata_reason": "Four intermediate BF16 nodes describe the actual two-stage graph. The earlier mids2 probe already passed. Exact replacement and raw-byte requirements remain mandatory.", + "log_sha256": { + "m4": "288fdd9dd60928f489a68a779db164399c203045f9e4efade9954f6fc6c5a1ff", + "m5": "acac3a048ebd0a35c7ec29f046cd48ae59deb5e85bcf29c4fb8050b60ef34d0b" + } + }, + "performance": { + "scope": "Same-machine old/new native HTTP ABBA. No new comparison against the standard runtime and no claim that the remaining concurrency target is met.", + "host": "M5 Ultra", + "checkpoint_bits": 4, + "context": 70000, + "parallel": 4, + "prompt_cache_gib": 0, + "temperature": 0, + "drafting": "default enabled", + "thinking": false, + "output_tokens_per_request": 256, + "repetitions_per_leg": 3, + "order": [ + "old", + "new", + "new", + "old" + ], + "measured_requests": 240, + "all_output_lengths_and_token_hashes_equal": true, + "decode": [ + { + "family": "text", + "prompt": "long-code-chat", + "streams": 1, + "old_tps": 323.37859207406774, + "new_tps": 340.56213946137177, + "throughput_gain_percent": 5.313755396451314, + "all_same_token_hash": true + }, + { + "family": "text", + "prompt": "long-code-chat", + "streams": 4, + "old_tps": 877.3614816835146, + "new_tps": 909.6937766053586, + "throughput_gain_percent": 3.6851737393124973, + "all_same_token_hash": true + }, + { + "family": "text", + "prompt": "long-prose-chat", + "streams": 1, + "old_tps": 338.05606841419115, + "new_tps": 355.5690832078852, + "throughput_gain_percent": 5.180505966317073, + "all_same_token_hash": true + }, + { + "family": "text", + "prompt": "long-prose-chat", + "streams": 4, + "old_tps": 897.5693493427952, + "new_tps": 928.0139771021795, + "throughput_gain_percent": 3.3918969917673847, + "all_same_token_hash": true + }, + { + "family": "vl", + "prompt": "long-code-chat", + "streams": 1, + "old_tps": 307.4211480014529, + "new_tps": 323.0724303307775, + "throughput_gain_percent": 5.091153432700946, + "all_same_token_hash": true + }, + { + "family": "vl", + "prompt": "long-code-chat", + "streams": 4, + "old_tps": 852.5923561448815, + "new_tps": 883.2251823336844, + "throughput_gain_percent": 3.5929041549602525, + "all_same_token_hash": true + }, + { + "family": "vl", + "prompt": "long-prose-chat", + "streams": 1, + "old_tps": 305.46397849556536, + "new_tps": 320.49747408261396, + "throughput_gain_percent": 4.921528116372276, + "all_same_token_hash": true + }, + { + "family": "vl", + "prompt": "long-prose-chat", + "streams": 4, + "old_tps": 848.7382388763339, + "new_tps": 879.6497949219245, + "throughput_gain_percent": 3.642060016821591, + "all_same_token_hash": true + } + ], + "cold_prefill": "Exact 2k/8k cached_tokens=0 exercised; tiny TTFT changes are not claimed as an improvement.", + "status_sha256": "83410ab791e78510de0e4a1980ea9b205bac36ae74f7b57503ea40f166e290d2" + }, + "applied_public_validation": { + "parent_commit": "6b318cedb9d9520ee53ee4a56abeb5d6ced5d461", + "scope": "Actual public checkout with only the reviewed fusion patch applied. The frozen six-quantization model/state/serving proofs above were not repeated because the runtime source hashes match.", + "source_sha256": { + "zig/kernels/metal/lfm3/ops.metal": "1659bb9185d966dd30276a6668465d25c06753a03c4cb38a02c3684514a657ae", + "zig/src/families/lfm3/forward.zig": "1ab4e29b47056c3232d3e8278c40c6b7dcf295d52d1f172e71ea47d95e92ff0f", + "zig/src/families/lfm3/kernels.zig": "7f04b485a2e9d1dc5ad6d2c7e49a4278ad61128fb946daee878a665e7ba21fed", + "zig/tests/contract/metal_lfm3_postnorm.zig": "8c362ca20a90fe093f1f08d65ace30affabe63879456ba0394ba84532d688594", + "zig/tests/contract/metal_main.zig": "3eff9d0bb4cfdbd0c990ed4201a094bcf91107e0e2ad5c4792551eda85db6922", + "zig/tests/contract/metal_limits.zig": "5d74368ae9b00d65373421719423ba98b6627d02645909d114e418ab9cc2a0e9", + "zig/src/core/contract/lfm3_records.zig": "33cf7203d5e371a016cceecb165923355a72dbcc4eadd1a3ceb6a92dbad0bd54" + }, + "build_command": "zig build native tf-kernel-contract dev-tools test-cuda-host -Dcpu=apple_m1 -Dversion=1.0.0 --summary all", + "build_exit_code": 0, + "build_summary": "Build Summary: 494/494 steps succeeded; 3429/3444 tests passed (15 skipped)", + "build_log_sha256": "1e4579b3d905dc834b6709210418e2a37f68769473ef402accd76b2d8b03d129", + "artifact_sha256": { + "zig-out/native/bin/tensorfold-native": "ad3ca5a0d39daa399b5bde058514291ec52f87a50799528127c654d30a139672", + "zig-out/bin/tf-kernel-contract": "0769ba295330784fffce2f7fd98f31ecf71edb44a3dbc55070863c3854d9b82b", + "zig-out/bin/tf-lfm3-forward": "8ef7afff7df80dda6d573c1c8c9b0079ecdfc0752fe35c827c4355d6cde549e2" + }, + "gpu_contract": { + "m4": { + "binary_sha256": "0769ba295330784fffce2f7fd98f31ecf71edb44a3dbc55070863c3854d9b82b", + "summary": "SUMMARY 1 kernels: 1 pass gates 1-4, 0 fail, 0 known findings; 1 controls: 1 red as planted, 0 missed; 0 limit problems; gates 27 ms, limits 388 ms", + "full_normal_launch_reflection": true, + "log_sha256": "98e12c1a886974cf94b52b4be60ed9f087b9c59f4570801715df1c70fa2a8467" + }, + "m5": { + "binary_sha256": "0769ba295330784fffce2f7fd98f31ecf71edb44a3dbc55070863c3854d9b82b", + "summary": "SUMMARY 1 kernels: 1 pass gates 1-4, 0 fail, 0 known findings; 1 controls: 1 red as planted, 0 missed; 0 limit problems; gates 15 ms, limits 120 ms", + "full_normal_launch_reflection": true, + "log_sha256": "993f191b92e0f69cd78e019260b4b1bf62d0c7881f938a77a84134ccd6244b1b" + } + }, + "format": "zig fmt --check passes for all six changed Zig files", + "lean": { + "base_findings": 428, + "applied_findings": 428, + "new_findings": 0 + }, + "not_claimed": "This combined host build is not a rerun of zig build test test-golden or the complete backend kernel numerical suite." + }, + "limits_source_selection_correction": { + "reason": "The original Gate 5 postnorm-bad record selected normal source because its fault macro was placed in the vision branch. The numerical removed-rounding control already used its fault macro correctly. Move the macro selection to the generic LFM3 branch and remove it from the vision branch. Runtime arithmetic is unchanged.", + "supersedes": "Only the earlier fault-variant launch-reflection source selection; all exact runtime/model proofs and normal/control numerical checks remain valid.", + "record": "lfm3 postnorm-bad lfm3_post_norm", + "macro": "L3_POSTNORM_SKIP_ROUND=1", + "variants": { + "main": { + "parent_commit": "1cdcd2c1d580f8a4b9bf3f9dec1984548097328e", + "source_selector_sha256": "3cefea93fa4d813d0b2e79fa781d2df4a07d91e150f9ecf4d3e6fb8b7216e684", + "previously_misselected_normal_source_sha256": "2a9edd589a50dea6c3289770d386459e14aa60c707e797e8e23d8f9fc3d73678", + "corrected_fault_source_sha256": "00068f4c021a65ef139e07ba9991f17a15ee3bc3a36a28d68cf88ae6fefa01f6", + "source_construction": "#define L3_POSTNORM_SKIP_ROUND 1 plus newline, ops.metal, newline, attention.metal", + "contract_binary_sha256": "dc23cf60560fd688f873bafb3fe2b31339b13af214d9489c65d9ff42aac1672b", + "build_exit_code": 0, + "build_summary": "Build Summary: 43/43 steps succeeded", + "checks": { + "m4": { + "summary": "SUMMARY 1 kernels: 1 pass gates 1-4, 0 fail, 0 known findings; 1 controls: 1 red as planted, 0 missed; 0 limit problems; gates 9 ms, limits 165 ms", + "full_normal_launch_reflection": true, + "normal_launch_records": 355, + "fault_reflection": { + "threads": 256, + "static_threadgroup_bytes": 144 + }, + "log_sha256": "20d7a6766a63fe4bd4d1262491947436e6c01a6a997c60703debd566df7963c0" + }, + "m5": { + "summary": "SUMMARY 1 kernels: 1 pass gates 1-4, 0 fail, 0 known findings; 1 controls: 1 red as planted, 0 missed; 0 limit problems; gates 9 ms, limits 9644 ms", + "full_normal_launch_reflection": true, + "normal_launch_records": 355, + "fault_reflection": { + "threads": 256, + "static_threadgroup_bytes": 144 + }, + "log_sha256": "da6c1ad478b9b91cd5bae7d44f9e176e86d7d6aeaea97db50f3b569899abd6d4" + } + } + }, + "text_foundation": { + "parent_commit": "fea79f5fad0c8e951ae1157fa245f244c68c492c", + "source_selector_sha256": "b23efe8d782cd45ba452375bab0d72ad297b5bd35e299ddb5ce4a959cea89b9a", + "previously_misselected_normal_source_sha256": "2a9edd589a50dea6c3289770d386459e14aa60c707e797e8e23d8f9fc3d73678", + "corrected_fault_source_sha256": "00068f4c021a65ef139e07ba9991f17a15ee3bc3a36a28d68cf88ae6fefa01f6", + "source_construction": "#define L3_POSTNORM_SKIP_ROUND 1 plus newline, ops.metal, newline, attention.metal", + "contract_binary_sha256": "69153461afe40f5f45d14fd838306b70b85e86d2c937a304a7e9b4d9adf9e249", + "build_exit_code": 0, + "build_summary": "Build Summary: 43/43 steps succeeded", + "checks": { + "m4": { + "summary": "SUMMARY 1 kernels: 1 pass gates 1-4, 0 fail, 0 known findings; 1 controls: 1 red as planted, 0 missed; 0 limit problems; gates 8 ms, limits 1137 ms", + "full_normal_launch_reflection": true, + "normal_launch_records": 337, + "fault_reflection": { + "threads": 256, + "static_threadgroup_bytes": 144 + }, + "log_sha256": "e39f929d7cc6eebd5b92e186cd8177d91b0a34d1ce8720e1fb6e915fb299f95d" + } + } + } + }, + "formatting": "zig fmt --check passes in main and text", + "model_repetition": "None: this correction changes contract source selection only." + } +} diff --git a/docs/receipts/lfm3/prefill-lifecycle.json b/docs/receipts/lfm3/prefill-lifecycle.json new file mode 100644 index 0000000000..02f9eedc82 --- /dev/null +++ b/docs/receipts/lfm3/prefill-lifecycle.json @@ -0,0 +1,87 @@ +{ + "scope": "Local M4, text and VL 4-bit, prefill cancellation/yield/progress ownership correctness; no performance or precision change", + "source_sha256": { + "zig/build/lfm3_host.zig": "2a8e1aec41ed4e43835b6bee7958fdd95f17ddf108c509eab0a88a9a3ec63089", + "zig/src/families/lfm3/backend.zig": "4ed5d33b869dc4be0d82d406cff4abc0f65a63687c45d8b3aaa61e26361db233", + "zig/src/families/lfm3/prefill_control.zig": "f34af732f2fbf77beb2608a1c220ab84579942ea4b0a4a663f16ba2ab9fcad68", + "zig/src/lfm3_prefill_tests.zig": "0946c1207c410b4ff767ac3de8b270e0ccfca5c62a228cecb1e2e147cb9eb4c6", + "zig/tests/lfm3_row_tune.zig": "443174a7ba48a569db49d709296c8ec9f00288f064f4f912a2ed27546693af55" + }, + "patch_sha256": "41eba72d2438b54c1a23a2f0ab0be1c4241a25eb5ed99504af870210949668bb", + "cpu_tests": { + "passed": 10, + "failed": 0, + "command": "zig build test-lfm3-prefill-host --summary all", + "scope": "Actual focused registered target reported 10 tests passing; five source-defined focused tests and their imported real Stream coverage.", + "build_log_sha256": "7c7206129986598a9279ea1ba9046ae2a555fac06a3708ff5e8292ef7014c83e" + }, + "actual_device": { + "untouched_old": { + "exit": 1, + "failure": "ExpectedInterruption", + "callback_checks": 0 + }, + "fixed": { + "exit": 0, + "cases_per_model": 4, + "models": 2, + "resumed_state_logits_draws": "exact", + "ownership": "clean" + } + }, + "diagnostic_excluded_from_patch": "Private actual-model callback diagnostic; immutable binary hashes below", + "evidence_sha256": { + "1.7B-after.log": "6d046fce8b7a213090872cdcc5d5a634358fb9254673dca0ba7369980921546b", + "events.json": "d29a50383a6ac6c86a1cecf58170d058b257f7ad9baca7b1c9f38199966e87fc", + "VL-2B-before.log": "70f184adaad6608a5c4f2ad2a4c528bb4ca43447a6162511329c6c0221c64a32", + "complete.json": "04b28fac6e93b10e83ff2ee9409e029aee6d20fb89ad69d35d6b0f2fd10b11e2", + "1.7B-before.log": "ddcfaf50992be7fdbce8b1e28e56553e294bb6de757c152d21839737e5f3ff92", + "VL-2B-after.log": "7e335597de283db894e2d1cfa22e835783d1abac1b11ef8ba692c0be27dec55d" + }, + "timings": "discarded; correctness only", + "binaries": { + "before": "54ee1e049e9daad368386795c0815c6ad40bb3b9dabaa88b23634be7003ef28f", + "after": "6d58b931d840c73eb6f6a037d8b5868d869584f7829a4c84044e4ec045f4e940" + }, + "host": { + "hardware": "Apple M4 Pro, 20 GPU cores", + "memory_bytes": 51539607552, + "os": "macOS 26.5.1, build 25F80" + }, + "lease": { + "path": "/tmp/tensorfold-model.lock", + "exclusive": true, + "same_alias_inode": true, + "single_model_admission": true, + "owned_children_exited": 4, + "released_after_children": true + }, + "cases": { + "text4": { + "after": [ + "CASE yield=false immediate=true observed=Cancelled cancel_checks=1 yield_checks=0 fill_at=0 caches=0 marks=0", + "CASE yield=false immediate=false observed=Cancelled cancel_checks=4 yield_checks=3 fill_at=256 caches=0 marks=2", + "CASE yield=true immediate=true observed=Yielded cancel_checks=1 yield_checks=1 fill_at=0 caches=0 marks=0", + "CASE yield=true immediate=false observed=Yielded cancel_checks=4 yield_checks=4 fill_at=256 caches=0 marks=2" + ], + "before": [ + "CASE yield=false immediate=true observed=success cancel_checks=0 yield_checks=0 fill_at=0 caches=1 marks=1" + ] + }, + "vl4": { + "after": [ + "CASE yield=false immediate=true observed=Cancelled cancel_checks=1 yield_checks=0 fill_at=0 caches=0 marks=0", + "CASE yield=false immediate=false observed=Cancelled cancel_checks=4 yield_checks=3 fill_at=256 caches=0 marks=2", + "CASE yield=true immediate=true observed=Yielded cancel_checks=1 yield_checks=1 fill_at=0 caches=0 marks=0", + "CASE yield=true immediate=false observed=Yielded cancel_checks=4 yield_checks=4 fill_at=256 caches=0 marks=2" + ], + "before": [ + "CASE yield=false immediate=true observed=success cancel_checks=0 yield_checks=0 fill_at=0 caches=1 marks=1" + ] + } + }, + "scope_limits": [ + "Final-chunk cancellation is covered by the host helper regression; device cases exercise immediate and partial cancellation/yield plus restored-prefix cancellation.", + "No image-input, numerical-fidelity, or performance gate is claimed." + ] +} diff --git a/docs/receipts/lfm3/probability-boundary-audit.json b/docs/receipts/lfm3/probability-boundary-audit.json new file mode 100644 index 0000000000..dd2eaba8b6 --- /dev/null +++ b/docs/receipts/lfm3/probability-boundary-audit.json @@ -0,0 +1,58 @@ +{ + "scope": "Primary-source probability activation boundary audit; no inference behavior or acceptance policy changed", + "official_eager_sources": [ + { + "model": "LFM3-1.7B", + "revision": "40ebe0aa6662c9d40389c2f5d565b6a2c6aebacd", + "path": "modeling_lfm3.py", + "sha256": "e8cd5254ea99ce6bd731b57b10908c2e8b5010ce8315341f895a38484885eeab", + "probability_cast_line": 196, + "source": "https://huggingface.co/LiquidAI/LFM3-1.7B/blob/40ebe0aa6662c9d40389c2f5d565b6a2c6aebacd/modeling_lfm3.py#L196", + "fact": "The eager implementation computes softmax in FP32, then explicitly casts probabilities to query dtype before the value matmul. The forward dispatch can select another attention backend." + }, + { + "model": "LFM3-VL-2B", + "revision": "f83d2b0a2296aaa277f394990a9b8b949ddf9691", + "path": "modeling_lfm3.py", + "sha256": "4a676bee2cf184d51a9dacbb9d7c4abae85515a042a920f8e2578b22bba3d6e8", + "probability_cast_line": 196, + "source": "https://huggingface.co/LiquidAI/LFM3-VL-2B/blob/f83d2b0a2296aaa277f394990a9b8b949ddf9691/modeling_lfm3.py#L196", + "fact": "The eager implementation computes softmax in FP32, then explicitly casts probabilities to query dtype before the value matmul. The forward dispatch can select another attention backend." + }, + { + "model": "LFM3-VL-2B", + "revision": "f83d2b0a2296aaa277f394990a9b8b949ddf9691", + "path": "modeling_lfm3_vl.py", + "sha256": "a41cbe7242f379808bcc005207ee0d7b1709528966c1e4da46baa7c41827ac79", + "probability_cast_line": 187, + "source": "https://huggingface.co/LiquidAI/LFM3-VL-2B/blob/f83d2b0a2296aaa277f394990a9b8b949ddf9691/modeling_lfm3_vl.py#L187", + "fact": "The eager implementation computes softmax in FP32, then explicitly casts probabilities to query dtype before the value matmul. The forward dispatch can select another attention backend." + } + ], + "pinned_mlx_vlm": { + "revision": "230fc308efb577d87446306bfb8a3e9f1a129558", + "decoder": "https://github.com/Blaizzy/mlx-vlm/blob/230fc308efb577d87446306bfb8a3e9f1a129558/mlx_vlm/models/lfm3/language.py#L93", + "vision": "https://github.com/Blaizzy/mlx-vlm/blob/230fc308efb577d87446306bfb8a3e9f1a129558/mlx_vlm/models/lfm3_vl/vision.py#L61", + "shared_dispatch": "https://github.com/Blaizzy/mlx-vlm/blob/230fc308efb577d87446306bfb8a3e9f1a129558/mlx_vlm/models/base.py#L446", + "fact": "Decoder and vision call fast SDPA without an explicit probability cast." + }, + "pinned_mlx_backend": { + "revision": "36969c094354086e15d2a327a9c50feb20b9cf11", + "vector": "https://github.com/ml-explore/mlx/blob/36969c094354086e15d2a327a9c50feb20b9cf11/mlx/backend/metal/kernels/sdpa_vector.h#L50", + "nax": "https://github.com/ml-explore/mlx/blob/36969c094354086e15d2a327a9c50feb20b9cf11/mlx/backend/metal/kernels/steel/attn/kernels/steel_attention_nax.h#L213", + "nax_mma": "https://github.com/ml-explore/mlx/blob/36969c094354086e15d2a327a9c50feb20b9cf11/mlx/backend/metal/kernels/steel/attn/nax.h#L405", + "dispatch": "https://github.com/ml-explore/mlx/blob/36969c094354086e15d2a327a9c50feb20b9cf11/mlx/backend/metal/scaled_dot_product_attention.cpp#L1507", + "facts": [ + "The vector path uses float score, probability and output accumulators and directly multiplies the FP32 exponential by values.", + "The M5 NAX path keeps its score/probability tile typed float and passes it to an MPP mixed-operand matmul with relaxed precision enabled; no explicit BF16 probability cast appears at the call site.", + "The GPU dispatch selects vector mode at query lengths at most eight." + ] + }, + "interpretation": [ + "BF16 probability storage is explicit in the official eager graph when query dtype is BF16.", + "Backend-dispatched SDPA does not establish one universal intermediate storage boundary across all valid implementations.", + "The current FP64 oracle is an ideal SDPA reference with named activation boundaries, not a faithful emulation of the eager BF16 graph.", + "An eager-graph variant must also account for query-dtype QK matmul/scaling outputs; changing only P is not proof of faithful eager arithmetic.", + "This audit does not authorize a BF16 precision cut in the current native FP32-probability baseline or waive the existing fidelity and performance gates." + ] +} diff --git a/docs/receipts/lfm3/projection-fp64-bounds.json b/docs/receipts/lfm3/projection-fp64-bounds.json new file mode 100644 index 0000000000..0206a54747 --- /dev/null +++ b/docs/receipts/lfm3/projection-fp64-bounds.json @@ -0,0 +1,13 @@ +{ + "cells": 66, + "outputs": 60954624, + "raw_tap_matches_all_native": true, + "one_row_tap_matches_all_native_first_rows": true, + "raw_FP32_bound_exceedances": 0, + "BF16_bound_exceedances": 0, + "max_raw_FP32_bound_fraction": 0.01465208540032716, + "max_BF16_bound_fraction": 0.9807553251550621, + "max_raw_abs_error": 2.555781975388527e-06, + "max_worst_column_raw_rms": 7.7385387436973e-07, + "model_acceptance_threshold": null +} diff --git a/docs/receipts/lfm3/projection-rounding-replay.json b/docs/receipts/lfm3/projection-rounding-replay.json new file mode 100644 index 0000000000..6427f176ab --- /dev/null +++ b/docs/receipts/lfm3/projection-rounding-replay.json @@ -0,0 +1,43 @@ +{ + "shape": [ + 257, + 6144 + ], + "projection": "language_model.model.layers.0.conv.in_proj", + "scope": "Projection rounding replay; not a whole-model fidelity pass", + "affine_fusion": "FP64 product-plus-add then FP32 round; checkpoint replay, not arbitrary IEEE fmaf proof", + "accepted_threshold": null, + "raw_equal": 1579008, + "values": 1579008, + "bf16_equal": 1579008, + "maximum_absolute_error": 0.0, + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "fixture": "reasoning-257", + "reference_module": "tools/lfm3/row_projection_reference.py", + "input_sha256": "c8be512c9fb15ede6f46052c3e6c467a61bfd333d0eb3f70fdb2d597a7a4fa40", + "native_raw_sha256": "0480f2bd0ad424db14bd7e197317909019ac0fbc413ff728382900b364febd0a", + "rounding_sites": [ + "Sequential FP32 additions of 16 BF16 inputs per lane", + "Four left-associated four-product sums, accumulated in FP32", + "Rounded bias-times-input-sum, fused scale-times-dot plus that product", + "FP32 accumulation across 512-input lane passes", + "FP32 XOR butterfly sum distances 1, 2, 4, 8, 16", + "BF16 RNE output storage" + ], + "explicit_fmaf_accelerator": { + "shape": [ + 257, + 6144 + ], + "projection": "language_model.model.layers.0.conv.in_proj", + "scope": "Projection rounding replay; not a whole-model fidelity pass", + "affine_fusion": "Explicit CPU fmaf", + "accepted_threshold": null, + "raw_equal": 1579008, + "values": 1579008, + "bf16_equal": 1579008, + "maximum_absolute_error": 0.0, + "source_sha256": "ff336f75fcf0a4f6c0ac92ed627bc1a6abb0b36b8f4cc95d876f2a8bb2f5cc42", + "library_sha256": "fd3bdcc07800ca3e4bda472228c5fc86afa1f8eab7c2ba4669b0eae1e07df82b" + } +} diff --git a/docs/receipts/lfm3/real-image-fp32-comparison.json b/docs/receipts/lfm3/real-image-fp32-comparison.json new file mode 100644 index 0000000000..dcb096a478 --- /dev/null +++ b/docs/receipts/lfm3/real-image-fp32-comparison.json @@ -0,0 +1,507 @@ +{ + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "phase": "prompt", + "scope": "Independent CPU FP32 graph diagnostics alongside retained required FP64; no Metal calls in FP32 graph", + "image": { + "url": "https://placecats.com/300/200", + "jpeg_sha256": "d1b324f529a96d8d8a16d973d88e1e974905d9c2c0e3e8b21a50b1c0e3fe2379", + "canonical_png_sha256": "005500792ee8ea6f35ac369d900fc788cd77667f145e7100c2cff57fae5355d9", + "conversion": "macOS sips JPEG to PNG; canonical PNG is the exact native/reference input" + }, + "grid": [ + 14, + 20 + ], + "image_start": 4, + "image_rows": 70, + "suffix_start": 74, + "suffix_rows": 16, + "all90_scope": "Contains70 image-placeholder rows,4 prefix rows and16 suffix rows; never conflate all90 with suffix16", + "modes": { + "fp32": "No internal BF16 activation boundaries; changes model arithmetic and is diagnostic only", + "fp32-bf16": "FP32 graph with the same explicit BF16 boundary sites as ideal FP64 graph; independent CPU reductions, not native lane replay", + "metal_assisted_replay": "Retained separately in real-image-rounding-reproduction.json; explicitly delegates scalar elementary functions to Metal" + }, + "fp32_semantics": { + "compute": "NumPy CPU float32 matmul/reductions/elementary functions; no Metal calls", + "boundaries": "fp32-bf16", + "dequantization": "FP32 q*scale+bias; separate multiply and add, no BF16 dequantized-weight cast", + "matmul_order": "NumPy BLAS implementation, not native GPU lane order", + "erf": "Binary64 libm erf on FP32 arguments then FP32 RNE", + "shape": [ + 90, + 128000 + ], + "numpy": "2.5.4", + "host": "arm64", + "blas": { + "Compilers": { + "c": { + "name": "clang", + "linker": "ld64", + "version": "15.0.0", + "commands": "cc" + }, + "cython": { + "name": "cython", + "linker": "cython", + "version": "3.3.0", + "commands": "cython" + }, + "c++": { + "name": "clang", + "linker": "ld64", + "version": "15.0.0", + "commands": "c++" + } + }, + "Machine Information": { + "host": { + "cpu": "aarch64", + "family": "aarch64", + "endian": "little", + "system": "darwin" + }, + "build": { + "cpu": "aarch64", + "family": "aarch64", + "endian": "little", + "system": "darwin" + } + }, + "Build Dependencies": { + "blas": { + "name": "accelerate", + "found": true, + "version": "unknown", + "detection method": "system", + "include directory": "unknown", + "lib directory": "unknown", + "openblas configuration": "unknown", + "pc file directory": "unknown" + }, + "lapack": { + "name": "accelerate", + "found": true, + "version": "unknown", + "detection method": "system", + "include directory": "unknown", + "lib directory": "unknown", + "openblas configuration": "unknown", + "pc file directory": "unknown" + } + }, + "Python Information": { + "path": "/private/var/folders/s6/5hzmn6lx4dz5nxs7k_0slzph0000gn/T/build-env-q3ulyt3x/bin/python", + "version": "3.13" + }, + "SIMD Extensions": { + "baseline": [ + "NEON", + "NEON_FP16", + "NEON_VFPV4", + "ASIMD" + ], + "found": [ + "ASIMDHP", + "ASIMDDP" + ], + "not found": [ + "ASIMDFHM" + ] + } + }, + "thread_environment": { + "OPENBLAS_NUM_THREADS": "2", + "VECLIB_MAXIMUM_THREADS": "2", + "OMP_NUM_THREADS": null + }, + "source_sha256": "6a8a7dc6434361f22a4da4e24821d14ee45ff2d31d57a9d4cbd5f8d1569f7f4e", + "accepted_threshold": null, + "fidelity_pass": null + }, + "results": { + "ideal_fp64_own_vision": { + "logits_sha256": "c34a303517fe4ea85a1886f9bc9928febebd96b677c1c9412462c5967c8fbc42", + "all90": { + "top1_differences": 7, + "disjoint_maximum_sets": 7, + "relative_rms": 0.022216577726492024, + "maximum_absolute_error": 1.59375 + }, + "suffix16": { + "top1_differences": 2, + "disjoint_maximum_sets": 2, + "relative_rms": 0.013412548767931996, + "maximum_absolute_error": 0.5 + }, + "suffix_differences": [ + { + "row": 81, + "native_top": 78, + "reference_top": 851, + "native_maximum_set": [ + 78 + ], + "reference_maximum_set": [ + 851 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.40625, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.40625 + }, + { + "row": 88, + "native_top": 309, + "reference_top": 22, + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0625 + } + ], + "disputed_rows": [ + { + "row": 81, + "contender_ids": [ + 78, + 851 + ], + "native_values": [ + 7.28125, + 7.15625 + ], + "reference_values": [ + 7.21875, + 7.625 + ], + "native_maximum_set": [ + 78 + ], + "reference_maximum_set": [ + 851 + ], + "reference_margin": 0.40625 + }, + { + "row": 88, + "contender_ids": [ + 22, + 309 + ], + "native_values": [ + 13.4375, + 13.5 + ], + "reference_values": [ + 13.4375, + 13.375 + ], + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22 + ], + "reference_margin": 0.0625 + } + ] + }, + "independent_fp32_own_vision": { + "logits_sha256": "13a0b9aa18f643ebba3b29bc71784fbe2244ca13e47a9632b0ac573df3e8353e", + "all90": { + "top1_differences": 6, + "disjoint_maximum_sets": 5, + "relative_rms": 0.029089294594932084, + "maximum_absolute_error": 1.75390625 + }, + "suffix16": { + "top1_differences": 2, + "disjoint_maximum_sets": 2, + "relative_rms": 0.01862716329656785, + "maximum_absolute_error": 0.859375 + }, + "suffix_differences": [ + { + "row": 81, + "native_top": 78, + "reference_top": 851, + "native_maximum_set": [ + 78 + ], + "reference_maximum_set": [ + 851 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.21875, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.21875 + }, + { + "row": 88, + "native_top": 309, + "reference_top": 22, + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.125, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.125 + } + ], + "disputed_rows": [ + { + "row": 81, + "contender_ids": [ + 78, + 851 + ], + "native_values": [ + 7.28125, + 7.15625 + ], + "reference_values": [ + 7.21875, + 7.4375 + ], + "native_maximum_set": [ + 78 + ], + "reference_maximum_set": [ + 851 + ], + "reference_margin": 0.21875 + }, + { + "row": 88, + "contender_ids": [ + 22, + 309 + ], + "native_values": [ + 13.4375, + 13.5 + ], + "reference_values": [ + 13.4375, + 13.3125 + ], + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22 + ], + "reference_margin": 0.125 + } + ] + }, + "independent_fp32_bf16_own_vision": { + "logits_sha256": "4e37c32710720c019db62d3d8fe574b202afc50ad6bdf19164c777e0d368e2e7", + "all90": { + "top1_differences": 5, + "disjoint_maximum_sets": 3, + "relative_rms": 0.018905629285201224, + "maximum_absolute_error": 0.8125 + }, + "suffix16": { + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.013024445100711624, + "maximum_absolute_error": 0.53125 + }, + "suffix_differences": [ + { + "row": 88, + "native_top": 309, + "reference_top": 22, + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22, + 309 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + } + ], + "disputed_rows": [ + { + "row": 81, + "contender_ids": [ + 78, + 851 + ], + "native_values": [ + 7.28125, + 7.15625 + ], + "reference_values": [ + 7.28125, + 7.15625 + ], + "native_maximum_set": [ + 78 + ], + "reference_maximum_set": [ + 78 + ], + "reference_margin": 0.125 + }, + { + "row": 88, + "contender_ids": [ + 22, + 309 + ], + "native_values": [ + 13.4375, + 13.5 + ], + "reference_values": [ + 13.5, + 13.5 + ], + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22, + 309 + ], + "reference_margin": 0.0 + } + ] + }, + "independent_fp32_bf16_native_vision": { + "logits_sha256": "c604db2c1e236eca9ba6197d5f464625e0c9bcc19a303aae0e12adc605977298", + "all90": { + "top1_differences": 3, + "disjoint_maximum_sets": 1, + "relative_rms": 0.015441009381036912, + "maximum_absolute_error": 0.8125 + }, + "suffix16": { + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.012171142512004731, + "maximum_absolute_error": 0.8125 + }, + "suffix_differences": [ + { + "row": 88, + "native_top": 309, + "reference_top": 22, + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22, + 309 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + } + ], + "disputed_rows": [ + { + "row": 81, + "contender_ids": [ + 78, + 851 + ], + "native_values": [ + 7.28125, + 7.15625 + ], + "reference_values": [ + 7.28125, + 7.25 + ], + "native_maximum_set": [ + 78 + ], + "reference_maximum_set": [ + 78 + ], + "reference_margin": 0.03125 + }, + { + "row": 88, + "contender_ids": [ + 22, + 309 + ], + "native_values": [ + 13.4375, + 13.5 + ], + "reference_values": [ + 13.5, + 13.5 + ], + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22, + 309 + ], + "reference_margin": 0.0 + } + ] + } + }, + "conclusion": "Pure FP32 retains both disputed suffix differences. Independent FP32 with model BF16 boundaries removes row81 difference and makes row88 a tied maximum overlapping native309. This demonstrates sensitivity to arithmetic/rounding; it does not establish a new acceptance threshold or attribute every difference to FP64 alone.", + "accepted_threshold": null, + "fidelity_pass": null, + "tests": { + "command": "python -m unittest discover -s tools/lfm3 -p test_fp32_graph_reference.py", + "passed": 6, + "failed": 0 + }, + "sources": { + "tools/lfm3/fp32_graph_reference.py": "6a8a7dc6434361f22a4da4e24821d14ee45ff2d31d57a9d4cbd5f8d1569f7f4e", + "tools/lfm3/test_fp32_graph_reference.py": "dd2c4fc06680dbab904c5d9d54ee0834fd2793f8d32c8c3d5cc9c231aad5468c", + "tools/lfm3/fp64_reference.py": "0fb1128ade095db87b6126792ab14930792ab1f7b41b998570356bc586f16b2e", + "tools/lfm3/compare_logits.py": "b1bf273c8f3ce89591bccb8f098c82763cddff11373dbe850302450fcee1502c" + }, + "input_sha256": { + "lfm3-real-cat-patches.npy": "f1c55a0dd28b91d9929f074995adadea79d7c0862c8cdcd0be6dbfda57383dc1", + "lfm3-real-cat-tokens.npy": "54c970db8c6ba82895cb2f18a48d397ea568ae885241129e9c5ee7dbbc91a24d", + "lfm3-real-cat-native-8.bf16": "aceb2b880177abfae718bdd660398b3cd7ee990d2dfcdce2a110d08dc4e33135" + }, + "raw_fp32_logits_sha256": { + "fp32-ownvision": "28d155c91aa531a7d832429ccbad4a397f0f8cc2a37795dffc3f534e62cc650c", + "fp32-bf16-ownvision-final": "3301aa73c1ff171de9b35d1bc5a64ff143d173f3d72edae4a387916b0e0c8d82", + "fp32-bf16-nativevision": "e0bcba9ec3e724bbd44e9e5d8b63510fa2d5d0dea0f8b24be8a7cdd3ad38b0d5" + }, + "commands": [ + "OPENBLAS_NUM_THREADS=2 VECLIB_MAXIMUM_THREADS=2 python tools/lfm3/fp32_graph_reference.py --model-dir MODEL --tokens TOKENS.npy --patches PATCHES.npy --grid-h 14 --grid-w 20 --image-start 4 --boundaries fp32 --output FP32.npy", + "Repeat with --boundaries fp32-bf16; fixed-native-image ablation replaces patch/grid arguments with --image-rows NATIVE_ROWS.npy" + ], + "native_identity_receipt": "real-image-rounding-reproduction.json", + "ideal_fp64_receipt": "real-image-fp64.json" +} diff --git a/docs/receipts/lfm3/real-image-fp64.json b/docs/receipts/lfm3/real-image-fp64.json new file mode 100644 index 0000000000..c870abf2eb --- /dev/null +++ b/docs/receipts/lfm3/real-image-fp64.json @@ -0,0 +1,4884 @@ +{ + "image": { + "url": "https://placecats.com/300/200", + "jpeg_sha256": "d1b324f529a96d8d8a16d973d88e1e974905d9c2c0e3e8b21a50b1c0e3fe2379", + "canonical_png_sha256": "005500792ee8ea6f35ac369d900fc788cd77667f145e7100c2cff57fae5355d9", + "conversion": "macOS sips JPEG to PNG; canonical PNG is the exact native/reference input" + }, + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "grid": [ + 14, + 20 + ], + "image_start": 4, + "image_rows": 70, + "suffix_start": 74, + "tokens": [ + 124899, + 5922, + 207, + 125009, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125010, + 90869, + 532, + 4145, + 22, + 124900, + 207, + 124899, + 63514, + 207, + 597, + 4145, + 5533, + 267, + 5140, + 22 + ], + "teacher_forced_suffix": "<|vision_end|>Describe this image.<|im_end|>\n<|im_start|>assistant\nThe image contains a cat.", + "patches_sha256": "f1c55a0dd28b91d9929f074995adadea79d7c0862c8cdcd0be6dbfda57383dc1", + "native_projected_rows_sha256": "ab2ac7db759e5c4a32fe7d60680ec47f7d1cbc5938fa7d8091bf4756bfeea4fc", + "fp64_projected_rows_sha256": "48e5056f330f9ce22e6f8a713f169c364a884d976f5d110abe9eeb2db3053719", + "token_fixture_sha256": "54c970db8c6ba82895cb2f18a48d397ea568ae885241129e9c5ee7dbbc91a24d", + "native_forward_diagnostic_sha256": "e80a2a14c0d8ed20b62088d0d8b87be4beb5fed3eb1a7336b3b00f8ac5f74bee", + "vision": { + "relative_rms": 0.01813030697564529, + "maximum_absolute_error": 0.009981155395507812, + "equal_values": 16235, + "values": 143360, + "worst_columns": [ + { + "column": 1990, + "rms": 0.0015688593706404128, + "max_abs": 0.009981155395507812 + }, + { + "column": 397, + "rms": 0.0015344000353975263, + "max_abs": 0.005859375 + }, + { + "column": 128, + "rms": 0.0014770298948466368, + "max_abs": 0.0087890625 + }, + { + "column": 1929, + "rms": 0.001300417453178259, + "max_abs": 0.0059814453125 + }, + { + "column": 873, + "rms": 0.0012845041962177839, + "max_abs": 0.005859375 + }, + { + "column": 11, + "rms": 0.0012795670765047575, + "max_abs": 0.00634765625 + }, + { + "column": 553, + "rms": 0.0012584769121040352, + "max_abs": 0.006011962890625 + }, + { + "column": 1610, + "rms": 0.001202713700111795, + "max_abs": 0.007080078125 + }, + { + "column": 1526, + "rms": 0.001193385050598642, + "max_abs": 0.005859375 + }, + { + "column": 1973, + "rms": 0.0011931334831220275, + "max_abs": 0.00630950927734375 + }, + { + "column": 725, + "rms": 0.0011622593543642388, + "max_abs": 0.0048828125 + }, + { + "column": 173, + "rms": 0.0011613064093054638, + "max_abs": 0.005859375 + }, + { + "column": 1764, + "rms": 0.0011519718060593134, + "max_abs": 0.00634765625 + }, + { + "column": 936, + "rms": 0.0011349096669917038, + "max_abs": 0.00634765625 + }, + { + "column": 276, + "rms": 0.001116903920039422, + "max_abs": 0.005615234375 + }, + { + "column": 297, + "rms": 0.0011168602727142719, + "max_abs": 0.0052490234375 + }, + { + "column": 1775, + "rms": 0.0011160820603579394, + "max_abs": 0.006103515625 + }, + { + "column": 444, + "rms": 0.0011153957723786742, + "max_abs": 0.00537109375 + }, + { + "column": 498, + "rms": 0.0011066090458204728, + "max_abs": 0.005859375 + }, + { + "column": 607, + "rms": 0.0011042805714237814, + "max_abs": 0.004638671875 + } + ], + "rows": [ + { + "row": 0, + "rms": 0.00038509225692759163, + "max_abs": 0.00146484375 + }, + { + "row": 1, + "rms": 0.0004149904938254954, + "max_abs": 0.001953125 + }, + { + "row": 2, + "rms": 0.000877077488810478, + "max_abs": 0.00390625 + }, + { + "row": 3, + "rms": 0.00047575119525369484, + "max_abs": 0.001953125 + }, + { + "row": 4, + "rms": 0.0005367853248696079, + "max_abs": 0.001953125 + }, + { + "row": 5, + "rms": 0.0005371665988300717, + "max_abs": 0.001953125 + }, + { + "row": 6, + "rms": 0.0005827084120754965, + "max_abs": 0.00244140625 + }, + { + "row": 7, + "rms": 0.0004190425252066865, + "max_abs": 0.002197265625 + }, + { + "row": 8, + "rms": 0.00042441074460589936, + "max_abs": 0.0015869140625 + }, + { + "row": 9, + "rms": 0.0003929156958412758, + "max_abs": 0.001678466796875 + }, + { + "row": 10, + "rms": 0.0007292396011436492, + "max_abs": 0.0029296875 + }, + { + "row": 11, + "rms": 0.0006779205369424231, + "max_abs": 0.0029296875 + }, + { + "row": 12, + "rms": 0.0004868173927818231, + "max_abs": 0.00244140625 + }, + { + "row": 13, + "rms": 0.00042974766525024277, + "max_abs": 0.0015716552734375 + }, + { + "row": 14, + "rms": 0.0007124745981844352, + "max_abs": 0.00341796875 + }, + { + "row": 15, + "rms": 0.0004814353117581487, + "max_abs": 0.001953125 + }, + { + "row": 16, + "rms": 0.0004405439544739124, + "max_abs": 0.001953125 + }, + { + "row": 17, + "rms": 0.0003959499697599191, + "max_abs": 0.00177001953125 + }, + { + "row": 18, + "rms": 0.0004506364623737153, + "max_abs": 0.002197265625 + }, + { + "row": 19, + "rms": 0.0006506666367911576, + "max_abs": 0.002685546875 + }, + { + "row": 20, + "rms": 0.0005829631963738181, + "max_abs": 0.00244140625 + }, + { + "row": 21, + "rms": 0.000537223958972607, + "max_abs": 0.001953125 + }, + { + "row": 22, + "rms": 0.00041393014316557734, + "max_abs": 0.00146484375 + }, + { + "row": 23, + "rms": 0.0005611598579519008, + "max_abs": 0.002593994140625 + }, + { + "row": 24, + "rms": 0.0005802262571670951, + "max_abs": 0.00244140625 + }, + { + "row": 25, + "rms": 0.0005058384695542535, + "max_abs": 0.001953125 + }, + { + "row": 26, + "rms": 0.0009398838259313429, + "max_abs": 0.004150390625 + }, + { + "row": 27, + "rms": 0.001931973033908195, + "max_abs": 0.009981155395507812 + }, + { + "row": 28, + "rms": 0.0006186816324889426, + "max_abs": 0.00244140625 + }, + { + "row": 29, + "rms": 0.0004461869618194747, + "max_abs": 0.001708984375 + }, + { + "row": 30, + "rms": 0.0005114067402734533, + "max_abs": 0.002582550048828125 + }, + { + "row": 31, + "rms": 0.0004927937150380757, + "max_abs": 0.001987457275390625 + }, + { + "row": 32, + "rms": 0.0007761428467895724, + "max_abs": 0.003173828125 + }, + { + "row": 33, + "rms": 0.00038593277389839835, + "max_abs": 0.00146484375 + }, + { + "row": 34, + "rms": 0.00044740458812283, + "max_abs": 0.001953125 + }, + { + "row": 35, + "rms": 0.0006860731967757307, + "max_abs": 0.002685546875 + }, + { + "row": 36, + "rms": 0.0006270084897137401, + "max_abs": 0.00244140625 + }, + { + "row": 37, + "rms": 0.0005387579256944436, + "max_abs": 0.00244140625 + }, + { + "row": 38, + "rms": 0.00046401603646580264, + "max_abs": 0.001953125 + }, + { + "row": 39, + "rms": 0.0012305790433560938, + "max_abs": 0.00439453125 + }, + { + "row": 40, + "rms": 0.0005861786870213448, + "max_abs": 0.001953125 + }, + { + "row": 41, + "rms": 0.00042216503025393007, + "max_abs": 0.00146484375 + }, + { + "row": 42, + "rms": 0.0013918158621081475, + "max_abs": 0.00634765625 + }, + { + "row": 43, + "rms": 0.00044729604682922894, + "max_abs": 0.001953125 + }, + { + "row": 44, + "rms": 0.00040878468728103377, + "max_abs": 0.001953125 + }, + { + "row": 45, + "rms": 0.0004457759460434441, + "max_abs": 0.001953125 + }, + { + "row": 46, + "rms": 0.00041377112596949927, + "max_abs": 0.00128173828125 + }, + { + "row": 47, + "rms": 0.00041814590378780754, + "max_abs": 0.0013885498046875 + }, + { + "row": 48, + "rms": 0.0004695319213416255, + "max_abs": 0.001953125 + }, + { + "row": 49, + "rms": 0.00048560295409770174, + "max_abs": 0.001953125 + }, + { + "row": 50, + "rms": 0.00041495053214236124, + "max_abs": 0.001708984375 + }, + { + "row": 51, + "rms": 0.0004966224103116107, + "max_abs": 0.00244140625 + }, + { + "row": 52, + "rms": 0.0006625663633869232, + "max_abs": 0.0029296875 + }, + { + "row": 53, + "rms": 0.0006223507282487358, + "max_abs": 0.00244140625 + }, + { + "row": 54, + "rms": 0.00044674795160498854, + "max_abs": 0.001708984375 + }, + { + "row": 55, + "rms": 0.00046771811420672177, + "max_abs": 0.001953125 + }, + { + "row": 56, + "rms": 0.0005680065193051587, + "max_abs": 0.0029296875 + }, + { + "row": 57, + "rms": 0.00041970000438700455, + "max_abs": 0.00146484375 + }, + { + "row": 58, + "rms": 0.00048188765321121625, + "max_abs": 0.001953125 + }, + { + "row": 59, + "rms": 0.0004281910620435689, + "max_abs": 0.001953125 + }, + { + "row": 60, + "rms": 0.0004889153776735873, + "max_abs": 0.001708984375 + }, + { + "row": 61, + "rms": 0.0017102720841656973, + "max_abs": 0.006591796875 + }, + { + "row": 62, + "rms": 0.00040234733626662625, + "max_abs": 0.00146484375 + }, + { + "row": 63, + "rms": 0.00042970885966766496, + "max_abs": 0.00164794921875 + }, + { + "row": 64, + "rms": 0.0008150417905341588, + "max_abs": 0.0048828125 + }, + { + "row": 65, + "rms": 0.000441042635922222, + "max_abs": 0.001953125 + }, + { + "row": 66, + "rms": 0.0005879597783345788, + "max_abs": 0.0029296875 + }, + { + "row": 67, + "rms": 0.0005276311619166364, + "max_abs": 0.001953125 + }, + { + "row": 68, + "rms": 0.00034564678098909714, + "max_abs": 0.001220703125 + }, + { + "row": 69, + "rms": 0.00045007914776163624, + "max_abs": 0.00146484375 + } + ] + }, + "decoder": { + "same_native_projected_rows": { + "rows": 90, + "vocabulary": 128000, + "top1_differences": 3, + "disjoint_maximum_sets": 0, + "relative_rms": 0.012972441485740975, + "maximum_absolute_error": 0.59228515625, + "differences": [ + { + "row": 15, + "native_top": 11876, + "reference_top": 3553, + "native_maximum_set": [ + 11876 + ], + "reference_maximum_set": [ + 3553, + 11876 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + }, + { + "row": 54, + "native_top": 84, + "reference_top": 266, + "native_maximum_set": [ + 84, + 266 + ], + "reference_maximum_set": [ + 266 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.03125, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.03125 + }, + { + "row": 88, + "native_top": 22, + "reference_top": 309, + "native_maximum_set": [ + 22, + 309 + ], + "reference_maximum_set": [ + 309 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + } + ], + "worst_rows": [ + { + "row": 0, + "rms_error": 0.14167088715067103, + "maximum_absolute_error": 0.59228515625 + }, + { + "row": 78, + "rms_error": 0.12248561069361408, + "maximum_absolute_error": 0.357421875 + }, + { + "row": 64, + "rms_error": 0.11182511017314119, + "maximum_absolute_error": 0.3125 + }, + { + "row": 24, + "rms_error": 0.0866794731728028, + "maximum_absolute_error": 0.375 + }, + { + "row": 3, + "rms_error": 0.07989438887416578, + "maximum_absolute_error": 0.328125 + }, + { + "row": 58, + "rms_error": 0.07935533339006284, + "maximum_absolute_error": 0.328125 + }, + { + "row": 66, + "rms_error": 0.07803778736849455, + "maximum_absolute_error": 0.20361328125 + }, + { + "row": 86, + "rms_error": 0.07341299374633652, + "maximum_absolute_error": 0.296875 + }, + { + "row": 14, + "rms_error": 0.07075159453364466, + "maximum_absolute_error": 0.3125 + }, + { + "row": 13, + "rms_error": 0.06974250803910013, + "maximum_absolute_error": 0.21875 + }, + { + "row": 74, + "rms_error": 0.06822229779840439, + "maximum_absolute_error": 0.244140625 + }, + { + "row": 42, + "rms_error": 0.06553193638682463, + "maximum_absolute_error": 0.203125 + }, + { + "row": 59, + "rms_error": 0.06538526477714782, + "maximum_absolute_error": 0.234375 + }, + { + "row": 84, + "rms_error": 0.062423116990238545, + "maximum_absolute_error": 0.34375 + }, + { + "row": 83, + "rms_error": 0.060912012323856096, + "maximum_absolute_error": 0.296875 + }, + { + "row": 72, + "rms_error": 0.06008451245252032, + "maximum_absolute_error": 0.34375 + }, + { + "row": 79, + "rms_error": 0.05938981342931649, + "maximum_absolute_error": 0.1875 + }, + { + "row": 89, + "rms_error": 0.05791377897695884, + "maximum_absolute_error": 0.3515625 + }, + { + "row": 34, + "rms_error": 0.05750644014736005, + "maximum_absolute_error": 0.421875 + }, + { + "row": 31, + "rms_error": 0.05736327192908228, + "maximum_absolute_error": 0.21875 + } + ], + "worst_columns": [ + { + "column": 283, + "rms_error": 0.09527168511786321, + "maximum_absolute_error": 0.421875 + }, + { + "column": 280, + "rms_error": 0.0923071313133541, + "maximum_absolute_error": 0.421875 + }, + { + "column": 267, + "rms_error": 0.08852095361666858, + "maximum_absolute_error": 0.328125 + }, + { + "column": 563, + "rms_error": 0.08818268299629918, + "maximum_absolute_error": 0.234375 + }, + { + "column": 1109, + "rms_error": 0.08758401111185665, + "maximum_absolute_error": 0.40625 + }, + { + "column": 462, + "rms_error": 0.08755840485056751, + "maximum_absolute_error": 0.546875 + }, + { + "column": 417, + "rms_error": 0.08731673136489858, + "maximum_absolute_error": 0.296875 + }, + { + "column": 495, + "rms_error": 0.08654889798245421, + "maximum_absolute_error": 0.353515625 + }, + { + "column": 767, + "rms_error": 0.086524835792439, + "maximum_absolute_error": 0.5 + }, + { + "column": 377, + "rms_error": 0.08625276324626419, + "maximum_absolute_error": 0.302734375 + }, + { + "column": 9101, + "rms_error": 0.08557000333842249, + "maximum_absolute_error": 0.421875 + }, + { + "column": 1628, + "rms_error": 0.08533500473923751, + "maximum_absolute_error": 0.296875 + }, + { + "column": 374, + "rms_error": 0.08485690552515096, + "maximum_absolute_error": 0.4375 + }, + { + "column": 526, + "rms_error": 0.08425616856225313, + "maximum_absolute_error": 0.25 + }, + { + "column": 2178, + "rms_error": 0.08412121337216075, + "maximum_absolute_error": 0.53125 + }, + { + "column": 1768, + "rms_error": 0.08407442775033669, + "maximum_absolute_error": 0.2421875 + }, + { + "column": 372, + "rms_error": 0.08339763212150117, + "maximum_absolute_error": 0.421875 + }, + { + "column": 37114, + "rms_error": 0.0832816768521584, + "maximum_absolute_error": 0.375 + }, + { + "column": 501, + "rms_error": 0.08297557329408449, + "maximum_absolute_error": 0.375 + }, + { + "column": 29061, + "rms_error": 0.08291168131609514, + "maximum_absolute_error": 0.34375 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "text_suffix": { + "rows": 16, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.011224499341193007, + "maximum_absolute_error": 0.357421875, + "differences": [ + { + "row": 88, + "native_top": 22, + "reference_top": 309, + "native_maximum_set": [ + 22, + 309 + ], + "reference_maximum_set": [ + 309 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + } + ], + "worst_rows": [ + { + "row": 78, + "rms_error": 0.12248561069361408, + "maximum_absolute_error": 0.357421875 + }, + { + "row": 86, + "rms_error": 0.07341299374633652, + "maximum_absolute_error": 0.296875 + }, + { + "row": 74, + "rms_error": 0.06822229779840439, + "maximum_absolute_error": 0.244140625 + }, + { + "row": 84, + "rms_error": 0.062423116990238545, + "maximum_absolute_error": 0.34375 + }, + { + "row": 83, + "rms_error": 0.060912012323856096, + "maximum_absolute_error": 0.296875 + }, + { + "row": 79, + "rms_error": 0.05938981342931649, + "maximum_absolute_error": 0.1875 + }, + { + "row": 89, + "rms_error": 0.05791377897695884, + "maximum_absolute_error": 0.3515625 + }, + { + "row": 87, + "rms_error": 0.045081485219578, + "maximum_absolute_error": 0.19140625 + }, + { + "row": 76, + "rms_error": 0.043838586231110865, + "maximum_absolute_error": 0.28125 + }, + { + "row": 82, + "rms_error": 0.04365357199063939, + "maximum_absolute_error": 0.21875 + }, + { + "row": 81, + "rms_error": 0.04223502380277463, + "maximum_absolute_error": 0.23046875 + }, + { + "row": 88, + "rms_error": 0.03854618539642888, + "maximum_absolute_error": 0.1875 + }, + { + "row": 85, + "rms_error": 0.03848121835927681, + "maximum_absolute_error": 0.203125 + }, + { + "row": 77, + "rms_error": 0.03779736863370633, + "maximum_absolute_error": 0.21875 + }, + { + "row": 80, + "rms_error": 0.03110414705779659, + "maximum_absolute_error": 0.1484375 + }, + { + "row": 75, + "rms_error": 0.029775309489710713, + "maximum_absolute_error": 0.125 + } + ], + "worst_columns": [ + { + "column": 2411, + "rms_error": 0.11533577547124645, + "maximum_absolute_error": 0.328125 + }, + { + "column": 1447, + "rms_error": 0.11317840379956114, + "maximum_absolute_error": 0.2724609375 + }, + { + "column": 9208, + "rms_error": 0.11293344708481622, + "maximum_absolute_error": 0.3125 + }, + { + "column": 5155, + "rms_error": 0.11266078056293939, + "maximum_absolute_error": 0.28125 + }, + { + "column": 10847, + "rms_error": 0.11233971091107102, + "maximum_absolute_error": 0.296875 + }, + { + "column": 4132, + "rms_error": 0.11224279875272572, + "maximum_absolute_error": 0.328125 + }, + { + "column": 9605, + "rms_error": 0.11075916799246054, + "maximum_absolute_error": 0.34375 + }, + { + "column": 377, + "rms_error": 0.1102499705204308, + "maximum_absolute_error": 0.302734375 + }, + { + "column": 20368, + "rms_error": 0.10977101160048115, + "maximum_absolute_error": 0.3125 + }, + { + "column": 117436, + "rms_error": 0.10911310721455798, + "maximum_absolute_error": 0.21875 + }, + { + "column": 47903, + "rms_error": 0.10889748261969855, + "maximum_absolute_error": 0.22380828857421875 + }, + { + "column": 1478, + "rms_error": 0.10859291570329246, + "maximum_absolute_error": 0.21875 + }, + { + "column": 29215, + "rms_error": 0.10844681493673235, + "maximum_absolute_error": 0.3125 + }, + { + "column": 10912, + "rms_error": 0.108442692701031, + "maximum_absolute_error": 0.296875 + }, + { + "column": 18121, + "rms_error": 0.10818267518848616, + "maximum_absolute_error": 0.25 + }, + { + "column": 2489, + "rms_error": 0.10812976980989382, + "maximum_absolute_error": 0.296875 + }, + { + "column": 39554, + "rms_error": 0.10798607071932075, + "maximum_absolute_error": 0.28125 + }, + { + "column": 29649, + "rms_error": 0.10794550114030071, + "maximum_absolute_error": 0.25 + }, + { + "column": 10955, + "rms_error": 0.1071552087839469, + "maximum_absolute_error": 0.28125 + }, + { + "column": 37134, + "rms_error": 0.10699182605717175, + "maximum_absolute_error": 0.3125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null + }, + "native_logits_sha256": "d2dc2c1014716cb9ead7543ee00562a1816bf50d4a18b927cd8d8151f0074d54", + "reference_logits_sha256": "ea2bb4dc9794ea8d5a6d714eaf2a5bf776713cd0a482f9fc754c22107b95f4a1" + }, + "same_fp64_projected_rows": { + "rows": 90, + "vocabulary": 128000, + "top1_differences": 3, + "disjoint_maximum_sets": 2, + "relative_rms": 0.014158856079061779, + "maximum_absolute_error": 0.59375, + "differences": [ + { + "row": 34, + "native_top": 5594, + "reference_top": 2911, + "native_maximum_set": [ + 5594 + ], + "reference_maximum_set": [ + 2911 + ], + "maximum_sets_overlap": false, + "native_margin": 0.15625, + "reference_margin": 0.03125, + "native_reference_contender_gap": 0.15625, + "reference_native_contender_gap": 0.03125 + }, + { + "row": 54, + "native_top": 266, + "reference_top": 84, + "native_maximum_set": [ + 266 + ], + "reference_maximum_set": [ + 84 + ], + "maximum_sets_overlap": false, + "native_margin": 0.046875, + "reference_margin": 0.03125, + "native_reference_contender_gap": 0.046875, + "reference_native_contender_gap": 0.03125 + }, + { + "row": 88, + "native_top": 309, + "reference_top": 22, + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22, + 309 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 64, + "rms_error": 0.14449757029683585, + "maximum_absolute_error": 0.4375 + }, + { + "row": 0, + "rms_error": 0.14167088715067103, + "maximum_absolute_error": 0.59228515625 + }, + { + "row": 14, + "rms_error": 0.12719449782472367, + "maximum_absolute_error": 0.4375 + }, + { + "row": 34, + "rms_error": 0.11917540280378529, + "maximum_absolute_error": 0.4609375 + }, + { + "row": 44, + "rms_error": 0.11621115329197179, + "maximum_absolute_error": 0.34375 + }, + { + "row": 73, + "rms_error": 0.10197229201591591, + "maximum_absolute_error": 0.25 + }, + { + "row": 39, + "rms_error": 0.08998475882118005, + "maximum_absolute_error": 0.3125 + }, + { + "row": 8, + "rms_error": 0.08916523433564291, + "maximum_absolute_error": 0.4296875 + }, + { + "row": 3, + "rms_error": 0.07989438887416578, + "maximum_absolute_error": 0.328125 + }, + { + "row": 24, + "rms_error": 0.07860293190705864, + "maximum_absolute_error": 0.4375 + }, + { + "row": 79, + "rms_error": 0.0780645262711307, + "maximum_absolute_error": 0.1875 + }, + { + "row": 66, + "rms_error": 0.07151927575284184, + "maximum_absolute_error": 0.2109375 + }, + { + "row": 33, + "rms_error": 0.07094835135644882, + "maximum_absolute_error": 0.390625 + }, + { + "row": 76, + "rms_error": 0.06821821862630469, + "maximum_absolute_error": 0.59375 + }, + { + "row": 26, + "rms_error": 0.06662915138667187, + "maximum_absolute_error": 0.21875 + }, + { + "row": 89, + "rms_error": 0.06513243354976006, + "maximum_absolute_error": 0.3125 + }, + { + "row": 84, + "rms_error": 0.06429891757179741, + "maximum_absolute_error": 0.34375 + }, + { + "row": 51, + "rms_error": 0.06312491717347178, + "maximum_absolute_error": 0.265625 + }, + { + "row": 86, + "rms_error": 0.06153170776406942, + "maximum_absolute_error": 0.34375 + }, + { + "row": 87, + "rms_error": 0.060788257552294694, + "maximum_absolute_error": 0.2578125 + } + ], + "worst_columns": [ + { + "column": 283, + "rms_error": 0.09870564171344597, + "maximum_absolute_error": 0.390625 + }, + { + "column": 518, + "rms_error": 0.09813089945580669, + "maximum_absolute_error": 0.3671875 + }, + { + "column": 501, + "rms_error": 0.09396304189946865, + "maximum_absolute_error": 0.375 + }, + { + "column": 4752, + "rms_error": 0.09302441762908323, + "maximum_absolute_error": 0.59375 + }, + { + "column": 346, + "rms_error": 0.09164530187222766, + "maximum_absolute_error": 0.3125 + }, + { + "column": 612, + "rms_error": 0.08995538130798408, + "maximum_absolute_error": 0.4296875 + }, + { + "column": 2115, + "rms_error": 0.08928488541778451, + "maximum_absolute_error": 0.3203125 + }, + { + "column": 1106, + "rms_error": 0.0890733963289641, + "maximum_absolute_error": 0.59228515625 + }, + { + "column": 1760, + "rms_error": 0.08897644797318778, + "maximum_absolute_error": 0.3125 + }, + { + "column": 339, + "rms_error": 0.088598853698158, + "maximum_absolute_error": 0.28125 + }, + { + "column": 1027, + "rms_error": 0.087713535296908, + "maximum_absolute_error": 0.3046875 + }, + { + "column": 267, + "rms_error": 0.0877015765076247, + "maximum_absolute_error": 0.28125 + }, + { + "column": 117877, + "rms_error": 0.08747986895916839, + "maximum_absolute_error": 0.375 + }, + { + "column": 58921, + "rms_error": 0.087227026192237, + "maximum_absolute_error": 0.3828125 + }, + { + "column": 117003, + "rms_error": 0.08713046029225263, + "maximum_absolute_error": 0.3125 + }, + { + "column": 14797, + "rms_error": 0.08692466871147085, + "maximum_absolute_error": 0.34375 + }, + { + "column": 563, + "rms_error": 0.0868222135878824, + "maximum_absolute_error": 0.4296875 + }, + { + "column": 376, + "rms_error": 0.08674096012461484, + "maximum_absolute_error": 0.3125 + }, + { + "column": 7575, + "rms_error": 0.08637582373145788, + "maximum_absolute_error": 0.34375 + }, + { + "column": 1894, + "rms_error": 0.08637283820473957, + "maximum_absolute_error": 0.3333740234375 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "text_suffix": { + "rows": 16, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.010381343967797625, + "maximum_absolute_error": 0.59375, + "differences": [ + { + "row": 88, + "native_top": 309, + "reference_top": 22, + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22, + 309 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 79, + "rms_error": 0.0780645262711307, + "maximum_absolute_error": 0.1875 + }, + { + "row": 76, + "rms_error": 0.06821821862630469, + "maximum_absolute_error": 0.59375 + }, + { + "row": 89, + "rms_error": 0.06513243354976006, + "maximum_absolute_error": 0.3125 + }, + { + "row": 84, + "rms_error": 0.06429891757179741, + "maximum_absolute_error": 0.34375 + }, + { + "row": 86, + "rms_error": 0.06153170776406942, + "maximum_absolute_error": 0.34375 + }, + { + "row": 87, + "rms_error": 0.060788257552294694, + "maximum_absolute_error": 0.2578125 + }, + { + "row": 82, + "rms_error": 0.059355582943006725, + "maximum_absolute_error": 0.21875 + }, + { + "row": 83, + "rms_error": 0.05373155297372426, + "maximum_absolute_error": 0.28125 + }, + { + "row": 81, + "rms_error": 0.053358733257710865, + "maximum_absolute_error": 0.3125 + }, + { + "row": 77, + "rms_error": 0.04273223154550211, + "maximum_absolute_error": 0.1875 + }, + { + "row": 74, + "rms_error": 0.041854330169046675, + "maximum_absolute_error": 0.28125 + }, + { + "row": 88, + "rms_error": 0.04003867483435802, + "maximum_absolute_error": 0.2109375 + }, + { + "row": 75, + "rms_error": 0.03820756413242102, + "maximum_absolute_error": 0.18359375 + }, + { + "row": 85, + "rms_error": 0.03565333800894985, + "maximum_absolute_error": 0.203125 + }, + { + "row": 80, + "rms_error": 0.03251769295110461, + "maximum_absolute_error": 0.17578125 + }, + { + "row": 78, + "rms_error": 0.0288727224827803, + "maximum_absolute_error": 0.15625 + } + ], + "worst_columns": [ + { + "column": 4752, + "rms_error": 0.17160446376418992, + "maximum_absolute_error": 0.59375 + }, + { + "column": 6440, + "rms_error": 0.12906706056088119, + "maximum_absolute_error": 0.462890625 + }, + { + "column": 18825, + "rms_error": 0.1261242289382533, + "maximum_absolute_error": 0.453125 + }, + { + "column": 10225, + "rms_error": 0.11869643220340088, + "maximum_absolute_error": 0.28515625 + }, + { + "column": 651, + "rms_error": 0.11832733538978837, + "maximum_absolute_error": 0.3671875 + }, + { + "column": 6767, + "rms_error": 0.11622837480784816, + "maximum_absolute_error": 0.28125 + }, + { + "column": 47960, + "rms_error": 0.11357132177356452, + "maximum_absolute_error": 0.3984375 + }, + { + "column": 612, + "rms_error": 0.1128463288622324, + "maximum_absolute_error": 0.390625 + }, + { + "column": 48756, + "rms_error": 0.1127538571934979, + "maximum_absolute_error": 0.34375 + }, + { + "column": 5027, + "rms_error": 0.11219749310543704, + "maximum_absolute_error": 0.3046875 + }, + { + "column": 9030, + "rms_error": 0.11147487738932582, + "maximum_absolute_error": 0.265625 + }, + { + "column": 108596, + "rms_error": 0.1114501953125, + "maximum_absolute_error": 0.359375 + }, + { + "column": 53596, + "rms_error": 0.11121777871531226, + "maximum_absolute_error": 0.34375 + }, + { + "column": 1027, + "rms_error": 0.11103219302609309, + "maximum_absolute_error": 0.3046875 + }, + { + "column": 95504, + "rms_error": 0.11086457770561231, + "maximum_absolute_error": 0.25 + }, + { + "column": 6373, + "rms_error": 0.11077663955456582, + "maximum_absolute_error": 0.25 + }, + { + "column": 48355, + "rms_error": 0.11025683254317267, + "maximum_absolute_error": 0.2837371826171875 + }, + { + "column": 32759, + "rms_error": 0.10972321800928234, + "maximum_absolute_error": 0.3046875 + }, + { + "column": 2080, + "rms_error": 0.10958406358676201, + "maximum_absolute_error": 0.296875 + }, + { + "column": 851, + "rms_error": 0.1091898304385418, + "maximum_absolute_error": 0.3125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null + }, + "native_logits_sha256": "7fcb749db23cf2e090441a5c6a97e925add9b81f34b79d6b15b9951099ddd358", + "reference_logits_sha256": "c16af1c1ff0095b2ced285886f4c54e21c48f244cb74b4fa615c8954163e620f" + }, + "end_to_end_own_vision_rows": { + "rows": 90, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 2, + "relative_rms": 0.02812434494869154, + "maximum_absolute_error": 2.5390625, + "differences": [ + { + "row": 34, + "native_top": 5594, + "reference_top": 2911, + "native_maximum_set": [ + 5594 + ], + "reference_maximum_set": [ + 2911 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.03125, + "native_reference_contender_gap": 0.25, + "reference_native_contender_gap": 0.03125 + }, + { + "row": 40, + "native_top": 4899, + "reference_top": 34541, + "native_maximum_set": [ + 4899 + ], + "reference_maximum_set": [ + 34541 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0625 + } + ], + "worst_rows": [ + { + "row": 31, + "rms_error": 0.6295615473865768, + "maximum_absolute_error": 2.5390625 + }, + { + "row": 14, + "rms_error": 0.4583900105693971, + "maximum_absolute_error": 1.375 + }, + { + "row": 44, + "rms_error": 0.2546304799623184, + "maximum_absolute_error": 1.06640625 + }, + { + "row": 0, + "rms_error": 0.14167088715067103, + "maximum_absolute_error": 0.59228515625 + }, + { + "row": 34, + "rms_error": 0.13362395908748093, + "maximum_absolute_error": 0.765625 + }, + { + "row": 43, + "rms_error": 0.1266629093436069, + "maximum_absolute_error": 0.8203125 + }, + { + "row": 72, + "rms_error": 0.12585469030246024, + "maximum_absolute_error": 0.609375 + }, + { + "row": 40, + "rms_error": 0.12564458204415568, + "maximum_absolute_error": 0.7265625 + }, + { + "row": 66, + "rms_error": 0.1251129267456409, + "maximum_absolute_error": 0.359375 + }, + { + "row": 67, + "rms_error": 0.11713658678492002, + "maximum_absolute_error": 0.4697265625 + }, + { + "row": 24, + "rms_error": 0.11604336419462331, + "maximum_absolute_error": 0.578125 + }, + { + "row": 39, + "rms_error": 0.1140450050053963, + "maximum_absolute_error": 0.74609375 + }, + { + "row": 30, + "rms_error": 0.10236631694704709, + "maximum_absolute_error": 0.5 + }, + { + "row": 65, + "rms_error": 0.10050247208171292, + "maximum_absolute_error": 0.59375 + }, + { + "row": 89, + "rms_error": 0.09796858084665877, + "maximum_absolute_error": 0.35546875 + }, + { + "row": 52, + "rms_error": 0.09755884491801062, + "maximum_absolute_error": 0.50390625 + }, + { + "row": 82, + "rms_error": 0.09612450835564357, + "maximum_absolute_error": 0.28125 + }, + { + "row": 17, + "rms_error": 0.09486707059144486, + "maximum_absolute_error": 0.791015625 + }, + { + "row": 41, + "rms_error": 0.09324326130837267, + "maximum_absolute_error": 0.4375 + }, + { + "row": 33, + "rms_error": 0.09087490275382837, + "maximum_absolute_error": 0.59375 + } + ], + "worst_columns": [ + { + "column": 42385, + "rms_error": 0.29791967617195436, + "maximum_absolute_error": 2.4921875 + }, + { + "column": 39091, + "rms_error": 0.28829632207176553, + "maximum_absolute_error": 2.5 + }, + { + "column": 24012, + "rms_error": 0.2849232457634144, + "maximum_absolute_error": 2.5390625 + }, + { + "column": 1427, + "rms_error": 0.2844398464190945, + "maximum_absolute_error": 2.497314453125 + }, + { + "column": 92335, + "rms_error": 0.2808404648778059, + "maximum_absolute_error": 2.515625 + }, + { + "column": 1420, + "rms_error": 0.2800456221061408, + "maximum_absolute_error": 2.421875 + }, + { + "column": 57782, + "rms_error": 0.2747600240470666, + "maximum_absolute_error": 2.390625 + }, + { + "column": 20473, + "rms_error": 0.2702198575419438, + "maximum_absolute_error": 2.375 + }, + { + "column": 12733, + "rms_error": 0.26863722024289205, + "maximum_absolute_error": 2.21875 + }, + { + "column": 23910, + "rms_error": 0.2666155450870272, + "maximum_absolute_error": 2.3984375 + }, + { + "column": 59969, + "rms_error": 0.26529832705909934, + "maximum_absolute_error": 2.2734375 + }, + { + "column": 95447, + "rms_error": 0.2624088133703063, + "maximum_absolute_error": 2.21875 + }, + { + "column": 55283, + "rms_error": 0.25920668200054897, + "maximum_absolute_error": 2.1953125 + }, + { + "column": 117147, + "rms_error": 0.2576033800531911, + "maximum_absolute_error": 2.1875 + }, + { + "column": 37940, + "rms_error": 0.2572721750003701, + "maximum_absolute_error": 2.15625 + }, + { + "column": 116308, + "rms_error": 0.25581059019091573, + "maximum_absolute_error": 2.296875 + }, + { + "column": 43854, + "rms_error": 0.2544676718929769, + "maximum_absolute_error": 2.21875 + }, + { + "column": 64319, + "rms_error": 0.2527230209271251, + "maximum_absolute_error": 2.140625 + }, + { + "column": 50696, + "rms_error": 0.2525461503880011, + "maximum_absolute_error": 2.203125 + }, + { + "column": 117785, + "rms_error": 0.2519815358285387, + "maximum_absolute_error": 2.046875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "text_suffix": { + "rows": 16, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.012445652348647095, + "maximum_absolute_error": 0.46875, + "differences": [], + "worst_rows": [ + { + "row": 89, + "rms_error": 0.09796858084665877, + "maximum_absolute_error": 0.35546875 + }, + { + "row": 82, + "rms_error": 0.09612450835564357, + "maximum_absolute_error": 0.28125 + }, + { + "row": 86, + "rms_error": 0.07995132116640384, + "maximum_absolute_error": 0.34375 + }, + { + "row": 83, + "rms_error": 0.07515822649991111, + "maximum_absolute_error": 0.359375 + }, + { + "row": 81, + "rms_error": 0.07160257402144439, + "maximum_absolute_error": 0.46875 + }, + { + "row": 74, + "rms_error": 0.07157990103346325, + "maximum_absolute_error": 0.3125 + }, + { + "row": 78, + "rms_error": 0.05800503213196821, + "maximum_absolute_error": 0.2548828125 + }, + { + "row": 84, + "rms_error": 0.05631889740571571, + "maximum_absolute_error": 0.3125 + }, + { + "row": 85, + "rms_error": 0.054296066515253046, + "maximum_absolute_error": 0.25 + }, + { + "row": 76, + "rms_error": 0.05188157615163868, + "maximum_absolute_error": 0.28125 + }, + { + "row": 75, + "rms_error": 0.05076188438843992, + "maximum_absolute_error": 0.17578125 + }, + { + "row": 79, + "rms_error": 0.0496180019266997, + "maximum_absolute_error": 0.1875 + }, + { + "row": 88, + "rms_error": 0.04745319760524986, + "maximum_absolute_error": 0.265625 + }, + { + "row": 87, + "rms_error": 0.04550976101171023, + "maximum_absolute_error": 0.28125 + }, + { + "row": 80, + "rms_error": 0.043724613188739235, + "maximum_absolute_error": 0.169921875 + }, + { + "row": 77, + "rms_error": 0.0324625974120783, + "maximum_absolute_error": 0.1875 + } + ], + "worst_columns": [ + { + "column": 37452, + "rms_error": 0.1330994143546939, + "maximum_absolute_error": 0.34375 + }, + { + "column": 59415, + "rms_error": 0.13228006787653943, + "maximum_absolute_error": 0.46875 + }, + { + "column": 27097, + "rms_error": 0.13115056222030894, + "maximum_absolute_error": 0.2578125 + }, + { + "column": 2779, + "rms_error": 0.1301745816037418, + "maximum_absolute_error": 0.34375 + }, + { + "column": 3273, + "rms_error": 0.1298497667077708, + "maximum_absolute_error": 0.34375 + }, + { + "column": 377, + "rms_error": 0.12820201745446932, + "maximum_absolute_error": 0.35400390625 + }, + { + "column": 6539, + "rms_error": 0.12667958059903367, + "maximum_absolute_error": 0.34375 + }, + { + "column": 118129, + "rms_error": 0.12592734624855367, + "maximum_absolute_error": 0.2890625 + }, + { + "column": 8034, + "rms_error": 0.12547308091305273, + "maximum_absolute_error": 0.3984375 + }, + { + "column": 1113, + "rms_error": 0.12535521580694153, + "maximum_absolute_error": 0.359375 + }, + { + "column": 750, + "rms_error": 0.12413704542277122, + "maximum_absolute_error": 0.3203125 + }, + { + "column": 58305, + "rms_error": 0.12365379215754528, + "maximum_absolute_error": 0.28125 + }, + { + "column": 3343, + "rms_error": 0.12325602119780095, + "maximum_absolute_error": 0.3984375 + }, + { + "column": 11325, + "rms_error": 0.12157516766520318, + "maximum_absolute_error": 0.28125 + }, + { + "column": 22267, + "rms_error": 0.12156536187183933, + "maximum_absolute_error": 0.234375 + }, + { + "column": 3040, + "rms_error": 0.12036702295414534, + "maximum_absolute_error": 0.34375 + }, + { + "column": 1270, + "rms_error": 0.1202560490536105, + "maximum_absolute_error": 0.37109375 + }, + { + "column": 417, + "rms_error": 0.12024440076643873, + "maximum_absolute_error": 0.34375 + }, + { + "column": 97969, + "rms_error": 0.1198787648999012, + "maximum_absolute_error": 0.35546875 + }, + { + "column": 38691, + "rms_error": 0.11936098965844703, + "maximum_absolute_error": 0.25 + } + ], + "accepted_threshold": null, + "fidelity_pass": null + }, + "native_logits_sha256": "d2dc2c1014716cb9ead7543ee00562a1816bf50d4a18b927cd8d8151f0074d54", + "reference_logits_sha256": "c16af1c1ff0095b2ced285886f4c54e21c48f244cb74b4fa615c8954163e620f" + } + }, + "accepted_threshold": null, + "fidelity_pass": null, + "vision_weight_identity": { + "4": { + "tensors": 138, + "tensor_bytes": 378760704, + "named_tensor_sha256": "4f4d0769ead2dc99fce516e6f43fb10be1070bbad4b708db8bb41df8bcb4c628" + }, + "6": { + "tensors": 138, + "tensor_bytes": 378760704, + "named_tensor_sha256": "40135b664ebd32cc67d662a6b4da767074fa4919668f1c711700f406340cc031" + }, + "8": { + "tensors": 138, + "tensor_bytes": 378760704, + "named_tensor_sha256": "40135b664ebd32cc67d662a6b4da767074fa4919668f1c711700f406340cc031" + } + }, + "other_widths": [ + { + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-6bit", + "vision": { + "relative_rms": 0.016063874280294436, + "maximum_absolute_error": 0.0087890625, + "equal_values": 16945, + "values": 143360, + "worst_columns": [ + { + "column": 498, + "rms": 0.0013091940942879653, + "max_abs": 0.0087890625 + }, + { + "column": 1764, + "rms": 0.001185183792016956, + "max_abs": 0.00732421875 + }, + { + "column": 809, + "rms": 0.0011589483218226793, + "max_abs": 0.00830078125 + }, + { + "column": 297, + "rms": 0.0011343321596087514, + "max_abs": 0.00714111328125 + }, + { + "column": 936, + "rms": 0.001095862149519839, + "max_abs": 0.00732421875 + }, + { + "column": 128, + "rms": 0.0010876887230548374, + "max_abs": 0.006103515625 + }, + { + "column": 652, + "rms": 0.0010590602510028337, + "max_abs": 0.005859375 + }, + { + "column": 1125, + "rms": 0.0010507872105445127, + "max_abs": 0.00604248046875 + }, + { + "column": 338, + "rms": 0.001043377334465457, + "max_abs": 0.00537109375 + }, + { + "column": 1960, + "rms": 0.0010385973306031866, + "max_abs": 0.00634765625 + }, + { + "column": 1608, + "rms": 0.0010359359747541973, + "max_abs": 0.0045166015625 + }, + { + "column": 1490, + "rms": 0.001009855121352609, + "max_abs": 0.00537109375 + }, + { + "column": 444, + "rms": 0.0010025326333805008, + "max_abs": 0.0068359375 + }, + { + "column": 397, + "rms": 0.0009856752557708178, + "max_abs": 0.00390625 + }, + { + "column": 1929, + "rms": 0.0009846070724547121, + "max_abs": 0.00390625 + }, + { + "column": 725, + "rms": 0.000984153556710703, + "max_abs": 0.00537109375 + }, + { + "column": 2022, + "rms": 0.0009788619755932626, + "max_abs": 0.0048828125 + }, + { + "column": 1777, + "rms": 0.000976186925169315, + "max_abs": 0.0048828125 + }, + { + "column": 1416, + "rms": 0.0009727661698073942, + "max_abs": 0.005615234375 + }, + { + "column": 1990, + "rms": 0.0009658278835947081, + "max_abs": 0.0057220458984375 + } + ], + "rows": [ + { + "row": 0, + "rms": 0.00038372740753499213, + "max_abs": 0.0013427734375 + }, + { + "row": 1, + "rms": 0.00038970468951474884, + "max_abs": 0.00146484375 + }, + { + "row": 2, + "rms": 0.0005385962238659673, + "max_abs": 0.00244140625 + }, + { + "row": 3, + "rms": 0.000532391804690437, + "max_abs": 0.001953125 + }, + { + "row": 4, + "rms": 0.00045059713743431107, + "max_abs": 0.001953125 + }, + { + "row": 5, + "rms": 0.000535412540394403, + "max_abs": 0.002197265625 + }, + { + "row": 6, + "rms": 0.0005446350280391706, + "max_abs": 0.001953125 + }, + { + "row": 7, + "rms": 0.00040900183839543433, + "max_abs": 0.001708984375 + }, + { + "row": 8, + "rms": 0.0005482143923531329, + "max_abs": 0.00244140625 + }, + { + "row": 9, + "rms": 0.0004320887828407833, + "max_abs": 0.0018310546875 + }, + { + "row": 10, + "rms": 0.0004493721978048441, + "max_abs": 0.001708984375 + }, + { + "row": 11, + "rms": 0.00048132567978884767, + "max_abs": 0.001953125 + }, + { + "row": 12, + "rms": 0.0004406089310737883, + "max_abs": 0.001735687255859375 + }, + { + "row": 13, + "rms": 0.00031486033651065233, + "max_abs": 0.00146484375 + }, + { + "row": 14, + "rms": 0.0006341479863918507, + "max_abs": 0.0029296875 + }, + { + "row": 15, + "rms": 0.0004002345030656259, + "max_abs": 0.00146484375 + }, + { + "row": 16, + "rms": 0.00046019922538941753, + "max_abs": 0.001953125 + }, + { + "row": 17, + "rms": 0.0004095935257686074, + "max_abs": 0.00146484375 + }, + { + "row": 18, + "rms": 0.0004174048032924595, + "max_abs": 0.001953125 + }, + { + "row": 19, + "rms": 0.0007863515636437885, + "max_abs": 0.00341796875 + }, + { + "row": 20, + "rms": 0.00046838318842117987, + "max_abs": 0.001953125 + }, + { + "row": 21, + "rms": 0.0004958759665262961, + "max_abs": 0.002197265625 + }, + { + "row": 22, + "rms": 0.0005967533694819817, + "max_abs": 0.00244140625 + }, + { + "row": 23, + "rms": 0.0004243509353734149, + "max_abs": 0.001708984375 + }, + { + "row": 24, + "rms": 0.0005357846031999338, + "max_abs": 0.0020599365234375 + }, + { + "row": 25, + "rms": 0.0004745263782153918, + "max_abs": 0.001708984375 + }, + { + "row": 26, + "rms": 0.0006049116600811196, + "max_abs": 0.00244140625 + }, + { + "row": 27, + "rms": 0.0012718290463919293, + "max_abs": 0.006103515625 + }, + { + "row": 28, + "rms": 0.0005543617429973317, + "max_abs": 0.00244140625 + }, + { + "row": 29, + "rms": 0.00046452820563499565, + "max_abs": 0.0020751953125 + }, + { + "row": 30, + "rms": 0.0004442953179747139, + "max_abs": 0.0019683837890625 + }, + { + "row": 31, + "rms": 0.00041361897628587705, + "max_abs": 0.001953125 + }, + { + "row": 32, + "rms": 0.0007000490247424031, + "max_abs": 0.0029296875 + }, + { + "row": 33, + "rms": 0.00047587498488368567, + "max_abs": 0.002197265625 + }, + { + "row": 34, + "rms": 0.00042651280049577295, + "max_abs": 0.00152587890625 + }, + { + "row": 35, + "rms": 0.0004061125662018534, + "max_abs": 0.0018310546875 + }, + { + "row": 36, + "rms": 0.0004777528468292542, + "max_abs": 0.001953125 + }, + { + "row": 37, + "rms": 0.0006343349672661181, + "max_abs": 0.0029296875 + }, + { + "row": 38, + "rms": 0.0004538810317093784, + "max_abs": 0.00146484375 + }, + { + "row": 39, + "rms": 0.0004983198958136023, + "max_abs": 0.001708984375 + }, + { + "row": 40, + "rms": 0.0005549457380957349, + "max_abs": 0.002197265625 + }, + { + "row": 41, + "rms": 0.000435093591768297, + "max_abs": 0.001953125 + }, + { + "row": 42, + "rms": 0.0016990532000404368, + "max_abs": 0.0087890625 + }, + { + "row": 43, + "rms": 0.0008343056505168412, + "max_abs": 0.00390625 + }, + { + "row": 44, + "rms": 0.0003863961261144731, + "max_abs": 0.0015869140625 + }, + { + "row": 45, + "rms": 0.0004130896401939031, + "max_abs": 0.001953125 + }, + { + "row": 46, + "rms": 0.0004656354475675182, + "max_abs": 0.001953125 + }, + { + "row": 47, + "rms": 0.00045687160490702494, + "max_abs": 0.001953125 + }, + { + "row": 48, + "rms": 0.0003940148584999622, + "max_abs": 0.001708984375 + }, + { + "row": 49, + "rms": 0.00042785081807080345, + "max_abs": 0.0015869140625 + }, + { + "row": 50, + "rms": 0.00042057304379851564, + "max_abs": 0.001708984375 + }, + { + "row": 51, + "rms": 0.0004346320529523064, + "max_abs": 0.00146484375 + }, + { + "row": 52, + "rms": 0.0007170450743457792, + "max_abs": 0.0040283203125 + }, + { + "row": 53, + "rms": 0.0005234081976803397, + "max_abs": 0.002197265625 + }, + { + "row": 54, + "rms": 0.0004300831551094984, + "max_abs": 0.001953125 + }, + { + "row": 55, + "rms": 0.00039691783537205294, + "max_abs": 0.001953125 + }, + { + "row": 56, + "rms": 0.0005015920315987764, + "max_abs": 0.002197265625 + }, + { + "row": 57, + "rms": 0.00047929273916284446, + "max_abs": 0.001953125 + }, + { + "row": 58, + "rms": 0.0005407952127570138, + "max_abs": 0.00225830078125 + }, + { + "row": 59, + "rms": 0.00042489098191947214, + "max_abs": 0.0015869140625 + }, + { + "row": 60, + "rms": 0.00039287798502104945, + "max_abs": 0.00146484375 + }, + { + "row": 61, + "rms": 0.0009851722974364375, + "max_abs": 0.003662109375 + }, + { + "row": 62, + "rms": 0.0005027295642356642, + "max_abs": 0.00209808349609375 + }, + { + "row": 63, + "rms": 0.0006288940981566789, + "max_abs": 0.002685546875 + }, + { + "row": 64, + "rms": 0.0006450404013174668, + "max_abs": 0.0029296875 + }, + { + "row": 65, + "rms": 0.00043091250461964635, + "max_abs": 0.001708984375 + }, + { + "row": 66, + "rms": 0.0005250592674429304, + "max_abs": 0.002197265625 + }, + { + "row": 67, + "rms": 0.000502115440446748, + "max_abs": 0.001953125 + }, + { + "row": 68, + "rms": 0.0006125769117026512, + "max_abs": 0.0029296875 + }, + { + "row": 69, + "rms": 0.00043968690926253284, + "max_abs": 0.001708984375 + } + ] + }, + "decoder": { + "rows": 90, + "vocabulary": 128000, + "top1_differences": 4, + "disjoint_maximum_sets": 3, + "relative_rms": 0.023126652059064244, + "maximum_absolute_error": 1.6484375, + "differences": [ + { + "row": 21, + "native_top": 415, + "reference_top": 20, + "native_maximum_set": [ + 415 + ], + "reference_maximum_set": [ + 20 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 24, + "native_top": 21, + "reference_top": 13279, + "native_maximum_set": [ + 21 + ], + "reference_maximum_set": [ + 13279 + ], + "maximum_sets_overlap": false, + "native_margin": 0.28125, + "reference_margin": 0.03125, + "native_reference_contender_gap": 0.28125, + "reference_native_contender_gap": 0.03125 + }, + { + "row": 52, + "native_top": 933, + "reference_top": 1628, + "native_maximum_set": [ + 933 + ], + "reference_maximum_set": [ + 1628 + ], + "maximum_sets_overlap": false, + "native_margin": 0.3125, + "reference_margin": 0.1875, + "native_reference_contender_gap": 0.3125, + "reference_native_contender_gap": 0.28125 + }, + { + "row": 54, + "native_top": 7832, + "reference_top": 1683, + "native_maximum_set": [ + 7832 + ], + "reference_maximum_set": [ + 1683, + 7832 + ], + "maximum_sets_overlap": true, + "native_margin": 0.03125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.03125, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 33, + "rms_error": 0.25663391866921725, + "maximum_absolute_error": 0.828125 + }, + { + "row": 31, + "rms_error": 0.2504493394251773, + "maximum_absolute_error": 1.6484375 + }, + { + "row": 14, + "rms_error": 0.22807760515265804, + "maximum_absolute_error": 0.65625 + }, + { + "row": 24, + "rms_error": 0.2067331342523365, + "maximum_absolute_error": 0.609375 + }, + { + "row": 53, + "rms_error": 0.17359424161214562, + "maximum_absolute_error": 0.5380859375 + }, + { + "row": 44, + "rms_error": 0.13956920251809574, + "maximum_absolute_error": 0.78125 + }, + { + "row": 86, + "rms_error": 0.12983974686456762, + "maximum_absolute_error": 0.46875 + }, + { + "row": 81, + "rms_error": 0.12732031587102097, + "maximum_absolute_error": 0.515625 + }, + { + "row": 17, + "rms_error": 0.12444584628664444, + "maximum_absolute_error": 0.767578125 + }, + { + "row": 3, + "rms_error": 0.12302304532129478, + "maximum_absolute_error": 0.4375 + }, + { + "row": 30, + "rms_error": 0.12011522304330144, + "maximum_absolute_error": 0.7890625 + }, + { + "row": 72, + "rms_error": 0.10987911844214017, + "maximum_absolute_error": 0.609375 + }, + { + "row": 40, + "rms_error": 0.1081011817491849, + "maximum_absolute_error": 0.5 + }, + { + "row": 47, + "rms_error": 0.10800964117851189, + "maximum_absolute_error": 0.359375 + }, + { + "row": 54, + "rms_error": 0.10544189612675053, + "maximum_absolute_error": 0.40625 + }, + { + "row": 19, + "rms_error": 0.10413301316411291, + "maximum_absolute_error": 0.40625 + }, + { + "row": 84, + "rms_error": 0.10350714849926548, + "maximum_absolute_error": 0.46875 + }, + { + "row": 39, + "rms_error": 0.10176695663134283, + "maximum_absolute_error": 0.609375 + }, + { + "row": 27, + "rms_error": 0.09448777791853481, + "maximum_absolute_error": 0.265625 + }, + { + "row": 67, + "rms_error": 0.0928782316503339, + "maximum_absolute_error": 0.375 + } + ], + "worst_columns": [ + { + "column": 461, + "rms_error": 0.19991099205938054, + "maximum_absolute_error": 1.6484375 + }, + { + "column": 417, + "rms_error": 0.18496020454674178, + "maximum_absolute_error": 1.3681640625 + }, + { + "column": 59610, + "rms_error": 0.18434468000289883, + "maximum_absolute_error": 1.44921875 + }, + { + "column": 82, + "rms_error": 0.1833022677816254, + "maximum_absolute_error": 1.53125 + }, + { + "column": 3273, + "rms_error": 0.1761274080059985, + "maximum_absolute_error": 1.3125 + }, + { + "column": 14998, + "rms_error": 0.1713850648612428, + "maximum_absolute_error": 1.359375 + }, + { + "column": 518, + "rms_error": 0.1710243000194276, + "maximum_absolute_error": 1.294921875 + }, + { + "column": 4834, + "rms_error": 0.17085718631917232, + "maximum_absolute_error": 1.375 + }, + { + "column": 104982, + "rms_error": 0.17012094502533545, + "maximum_absolute_error": 1.2763671875 + }, + { + "column": 4425, + "rms_error": 0.16890239819794733, + "maximum_absolute_error": 1.28125 + }, + { + "column": 495, + "rms_error": 0.1675999450359369, + "maximum_absolute_error": 1.25 + }, + { + "column": 15446, + "rms_error": 0.16411874604219687, + "maximum_absolute_error": 1.234375 + }, + { + "column": 5190, + "rms_error": 0.163092920334055, + "maximum_absolute_error": 1.375 + }, + { + "column": 7427, + "rms_error": 0.16277933036264844, + "maximum_absolute_error": 1.185546875 + }, + { + "column": 96983, + "rms_error": 0.1609604969280825, + "maximum_absolute_error": 1.046875 + }, + { + "column": 48589, + "rms_error": 0.15968040757459365, + "maximum_absolute_error": 1.140625 + }, + { + "column": 48388, + "rms_error": 0.15955504177427557, + "maximum_absolute_error": 1.125 + }, + { + "column": 6787, + "rms_error": 0.1585607768975797, + "maximum_absolute_error": 1.03125 + }, + { + "column": 58404, + "rms_error": 0.15825182004407265, + "maximum_absolute_error": 1.21875 + }, + { + "column": 116390, + "rms_error": 0.15777719982060762, + "maximum_absolute_error": 1.015625 + } + ], + "accepted_threshold": null, + "fidelity_pass": null + }, + "text_suffix": { + "rows": 16, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.015325597943788842, + "maximum_absolute_error": 0.515625, + "differences": [], + "worst_rows": [ + { + "row": 86, + "rms_error": 0.12983974686456762, + "maximum_absolute_error": 0.46875 + }, + { + "row": 81, + "rms_error": 0.12732031587102097, + "maximum_absolute_error": 0.515625 + }, + { + "row": 84, + "rms_error": 0.10350714849926548, + "maximum_absolute_error": 0.46875 + }, + { + "row": 80, + "rms_error": 0.08087380862335725, + "maximum_absolute_error": 0.21875 + }, + { + "row": 79, + "rms_error": 0.07965964158543284, + "maximum_absolute_error": 0.21875 + }, + { + "row": 78, + "rms_error": 0.07529425369285742, + "maximum_absolute_error": 0.26171875 + }, + { + "row": 76, + "rms_error": 0.07389853471728333, + "maximum_absolute_error": 0.431640625 + }, + { + "row": 83, + "rms_error": 0.07384908414923878, + "maximum_absolute_error": 0.34375 + }, + { + "row": 87, + "rms_error": 0.061938461077009746, + "maximum_absolute_error": 0.390625 + }, + { + "row": 89, + "rms_error": 0.06133959736469142, + "maximum_absolute_error": 0.390625 + }, + { + "row": 82, + "rms_error": 0.05947057960749595, + "maximum_absolute_error": 0.25 + }, + { + "row": 85, + "rms_error": 0.05941954177963855, + "maximum_absolute_error": 0.296875 + }, + { + "row": 88, + "rms_error": 0.05597531643539776, + "maximum_absolute_error": 0.26953125 + }, + { + "row": 77, + "rms_error": 0.04259418011083765, + "maximum_absolute_error": 0.234375 + }, + { + "row": 74, + "rms_error": 0.038931492926839055, + "maximum_absolute_error": 0.2578125 + }, + { + "row": 75, + "rms_error": 0.02554963560675411, + "maximum_absolute_error": 0.140625 + } + ], + "worst_columns": [ + { + "column": 4752, + "rms_error": 0.1596762774577553, + "maximum_absolute_error": 0.515625 + }, + { + "column": 49109, + "rms_error": 0.15685916410360362, + "maximum_absolute_error": 0.4375 + }, + { + "column": 64674, + "rms_error": 0.14875393630377298, + "maximum_absolute_error": 0.34375 + }, + { + "column": 1391, + "rms_error": 0.14757568745091898, + "maximum_absolute_error": 0.4375 + }, + { + "column": 96199, + "rms_error": 0.14605732019892534, + "maximum_absolute_error": 0.3125 + }, + { + "column": 3568, + "rms_error": 0.14581880042914133, + "maximum_absolute_error": 0.359375 + }, + { + "column": 13881, + "rms_error": 0.14559626610967097, + "maximum_absolute_error": 0.3125 + }, + { + "column": 91948, + "rms_error": 0.14517645261053228, + "maximum_absolute_error": 0.46875 + }, + { + "column": 2024, + "rms_error": 0.14501870922536383, + "maximum_absolute_error": 0.4375 + }, + { + "column": 9612, + "rms_error": 0.1449134515746058, + "maximum_absolute_error": 0.34375 + }, + { + "column": 45695, + "rms_error": 0.1447949452566456, + "maximum_absolute_error": 0.4375 + }, + { + "column": 60332, + "rms_error": 0.14395273259369343, + "maximum_absolute_error": 0.46875 + }, + { + "column": 10898, + "rms_error": 0.14387321167471606, + "maximum_absolute_error": 0.40625 + }, + { + "column": 96105, + "rms_error": 0.14379033061957783, + "maximum_absolute_error": 0.375 + }, + { + "column": 117881, + "rms_error": 0.14315553564306868, + "maximum_absolute_error": 0.4375 + }, + { + "column": 55559, + "rms_error": 0.14288547779821784, + "maximum_absolute_error": 0.3125 + }, + { + "column": 33192, + "rms_error": 0.1427920056752675, + "maximum_absolute_error": 0.375 + }, + { + "column": 20368, + "rms_error": 0.14264854886680386, + "maximum_absolute_error": 0.40625 + }, + { + "column": 33131, + "rms_error": 0.14263162523640444, + "maximum_absolute_error": 0.2890625 + }, + { + "column": 63561, + "rms_error": 0.14175974462206292, + "maximum_absolute_error": 0.34375 + } + ], + "accepted_threshold": null, + "fidelity_pass": null + }, + "native_projected_rows_sha256": "aceb2b880177abfae718bdd660398b3cd7ee990d2dfcdce2a110d08dc4e33135", + "fp64_projected_rows_sha256": "bc435e6390349c8be4a8ff12102ab941036ea1b13f0a1b4b9f1222189dee97a3", + "native_logits_sha256": "35336ed8641fd2f93ffa3af62b1b9ebeb3c1de39a17d138a2021fc2fa297360e", + "fp64_logits_sha256": "5e080e150047d502ff7b944ea0049a03daed405aa8cf31efce392272760cf00d", + "native_forward_diagnostic_sha256": "e80a2a14c0d8ed20b62088d0d8b87be4beb5fed3eb1a7336b3b00f8ac5f74bee", + "native_vision_diagnostic_sha256": "8bffb39af3c87626457ff020f91a6b76d48d1b795dd918578ccca61cb2743292", + "fp64_vision_scope": "6-bit checkpoint evaluated directly; 8-bit shares all 138 vision/projector tensors byte-for-byte, verified named-tensor hash", + "same_fp64_projected_rows_decoder": { + "rows": 90, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 1, + "relative_rms": 0.016445311311306988, + "maximum_absolute_error": 0.59375, + "differences": [ + { + "row": 24, + "native_top": 21, + "reference_top": 13279, + "native_maximum_set": [ + 21 + ], + "reference_maximum_set": [ + 13279 + ], + "maximum_sets_overlap": false, + "native_margin": 0.03125, + "reference_margin": 0.03125, + "native_reference_contender_gap": 0.03125, + "reference_native_contender_gap": 0.03125 + }, + { + "row": 54, + "native_top": 7832, + "reference_top": 1683, + "native_maximum_set": [ + 7832 + ], + "reference_maximum_set": [ + 1683, + 7832 + ], + "maximum_sets_overlap": true, + "native_margin": 0.078125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.078125, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 14, + "rms_error": 0.23335422208094994, + "maximum_absolute_error": 0.59375 + }, + { + "row": 83, + "rms_error": 0.17749967075621031, + "maximum_absolute_error": 0.46875 + }, + { + "row": 84, + "rms_error": 0.1272844447586001, + "maximum_absolute_error": 0.5 + }, + { + "row": 3, + "rms_error": 0.12302304532129478, + "maximum_absolute_error": 0.4375 + }, + { + "row": 80, + "rms_error": 0.10631625947983828, + "maximum_absolute_error": 0.28125 + }, + { + "row": 86, + "rms_error": 0.10152877501009015, + "maximum_absolute_error": 0.453125 + }, + { + "row": 89, + "rms_error": 0.09493442776255652, + "maximum_absolute_error": 0.40625 + }, + { + "row": 44, + "rms_error": 0.09130987267663834, + "maximum_absolute_error": 0.34375 + }, + { + "row": 24, + "rms_error": 0.08588515245019804, + "maximum_absolute_error": 0.46875 + }, + { + "row": 28, + "rms_error": 0.08180282075766566, + "maximum_absolute_error": 0.25 + }, + { + "row": 32, + "rms_error": 0.07686583919272973, + "maximum_absolute_error": 0.25 + }, + { + "row": 31, + "rms_error": 0.07635722546283522, + "maximum_absolute_error": 0.28125 + }, + { + "row": 4, + "rms_error": 0.07521725805367227, + "maximum_absolute_error": 0.28125 + }, + { + "row": 64, + "rms_error": 0.07498552052910223, + "maximum_absolute_error": 0.28125 + }, + { + "row": 67, + "rms_error": 0.06996710749201493, + "maximum_absolute_error": 0.25 + }, + { + "row": 53, + "rms_error": 0.06919764449821392, + "maximum_absolute_error": 0.296875 + }, + { + "row": 39, + "rms_error": 0.06917089851263995, + "maximum_absolute_error": 0.267578125 + }, + { + "row": 22, + "rms_error": 0.067956349165196, + "maximum_absolute_error": 0.203125 + }, + { + "row": 34, + "rms_error": 0.06539613589726158, + "maximum_absolute_error": 0.33203125 + }, + { + "row": 2, + "rms_error": 0.06523296096973401, + "maximum_absolute_error": 0.375 + } + ], + "worst_columns": [ + { + "column": 34701, + "rms_error": 0.09756440993986518, + "maximum_absolute_error": 0.46875 + }, + { + "column": 91238, + "rms_error": 0.09641875774101924, + "maximum_absolute_error": 0.4375 + }, + { + "column": 88618, + "rms_error": 0.09508529849171118, + "maximum_absolute_error": 0.53125 + }, + { + "column": 34398, + "rms_error": 0.09468930485078556, + "maximum_absolute_error": 0.40625 + }, + { + "column": 66150, + "rms_error": 0.0934733087086997, + "maximum_absolute_error": 0.34375 + }, + { + "column": 49653, + "rms_error": 0.09299049728592232, + "maximum_absolute_error": 0.375 + }, + { + "column": 18084, + "rms_error": 0.09288336415062441, + "maximum_absolute_error": 0.40625 + }, + { + "column": 116532, + "rms_error": 0.0925700781964981, + "maximum_absolute_error": 0.40625 + }, + { + "column": 267, + "rms_error": 0.09249979632014398, + "maximum_absolute_error": 0.375 + }, + { + "column": 43858, + "rms_error": 0.09174326961545844, + "maximum_absolute_error": 0.375 + }, + { + "column": 30322, + "rms_error": 0.09169983209674115, + "maximum_absolute_error": 0.46875 + }, + { + "column": 98310, + "rms_error": 0.09156085756786776, + "maximum_absolute_error": 0.40625 + }, + { + "column": 106635, + "rms_error": 0.0914539182523448, + "maximum_absolute_error": 0.40625 + }, + { + "column": 80459, + "rms_error": 0.09144088966355923, + "maximum_absolute_error": 0.4375 + }, + { + "column": 95446, + "rms_error": 0.09129318678504514, + "maximum_absolute_error": 0.40625 + }, + { + "column": 1631, + "rms_error": 0.09087311188442278, + "maximum_absolute_error": 0.34375 + }, + { + "column": 42751, + "rms_error": 0.09057556499450402, + "maximum_absolute_error": 0.375 + }, + { + "column": 1027, + "rms_error": 0.09049243014842652, + "maximum_absolute_error": 0.44921875 + }, + { + "column": 92417, + "rms_error": 0.09043060148476197, + "maximum_absolute_error": 0.40625 + }, + { + "column": 66748, + "rms_error": 0.09037738749643864, + "maximum_absolute_error": 0.59375 + } + ], + "accepted_threshold": null, + "fidelity_pass": null + }, + "same_fp64_projected_rows_text_suffix": { + "rows": 16, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.0157726673324293, + "maximum_absolute_error": 0.5, + "differences": [], + "worst_rows": [ + { + "row": 83, + "rms_error": 0.17749967075621031, + "maximum_absolute_error": 0.46875 + }, + { + "row": 84, + "rms_error": 0.1272844447586001, + "maximum_absolute_error": 0.5 + }, + { + "row": 80, + "rms_error": 0.10631625947983828, + "maximum_absolute_error": 0.28125 + }, + { + "row": 86, + "rms_error": 0.10152877501009015, + "maximum_absolute_error": 0.453125 + }, + { + "row": 89, + "rms_error": 0.09493442776255652, + "maximum_absolute_error": 0.40625 + }, + { + "row": 81, + "rms_error": 0.06358537069639142, + "maximum_absolute_error": 0.265625 + }, + { + "row": 87, + "rms_error": 0.05635314441037343, + "maximum_absolute_error": 0.3125 + }, + { + "row": 76, + "rms_error": 0.05541900052990697, + "maximum_absolute_error": 0.3203125 + }, + { + "row": 88, + "rms_error": 0.049226990539085934, + "maximum_absolute_error": 0.375 + }, + { + "row": 79, + "rms_error": 0.047860458528819406, + "maximum_absolute_error": 0.1640625 + }, + { + "row": 85, + "rms_error": 0.04425598777326575, + "maximum_absolute_error": 0.265625 + }, + { + "row": 78, + "rms_error": 0.03826662891740618, + "maximum_absolute_error": 0.25 + }, + { + "row": 82, + "rms_error": 0.03551484449024613, + "maximum_absolute_error": 0.21875 + }, + { + "row": 74, + "rms_error": 0.034773345335617226, + "maximum_absolute_error": 0.375 + }, + { + "row": 77, + "rms_error": 0.029109292444103235, + "maximum_absolute_error": 0.140625 + }, + { + "row": 75, + "rms_error": 0.02417622630874152, + "maximum_absolute_error": 0.1328125 + } + ], + "worst_columns": [ + { + "column": 34472, + "rms_error": 0.16273469848893676, + "maximum_absolute_error": 0.453125 + }, + { + "column": 59458, + "rms_error": 0.15054273962924325, + "maximum_absolute_error": 0.40625 + }, + { + "column": 5797, + "rms_error": 0.14836219081848054, + "maximum_absolute_error": 0.4375 + }, + { + "column": 18657, + "rms_error": 0.14769647825442367, + "maximum_absolute_error": 0.453125 + }, + { + "column": 14200, + "rms_error": 0.14727124271757738, + "maximum_absolute_error": 0.4375 + }, + { + "column": 50759, + "rms_error": 0.1469523773904428, + "maximum_absolute_error": 0.34375 + }, + { + "column": 15292, + "rms_error": 0.1465364490773648, + "maximum_absolute_error": 0.3125 + }, + { + "column": 34398, + "rms_error": 0.14522899566422892, + "maximum_absolute_error": 0.40625 + }, + { + "column": 49827, + "rms_error": 0.1450745962902173, + "maximum_absolute_error": 0.40625 + }, + { + "column": 49109, + "rms_error": 0.14338601511885682, + "maximum_absolute_error": 0.375 + }, + { + "column": 117325, + "rms_error": 0.14302318818565557, + "maximum_absolute_error": 0.375 + }, + { + "column": 117468, + "rms_error": 0.14282956879030756, + "maximum_absolute_error": 0.296875 + }, + { + "column": 6299, + "rms_error": 0.14261825205881784, + "maximum_absolute_error": 0.375 + }, + { + "column": 66390, + "rms_error": 0.14256474680386452, + "maximum_absolute_error": 0.375 + }, + { + "column": 41164, + "rms_error": 0.14247106425653816, + "maximum_absolute_error": 0.375 + }, + { + "column": 45033, + "rms_error": 0.1420957164271785, + "maximum_absolute_error": 0.375 + }, + { + "column": 17863, + "rms_error": 0.14177319878093364, + "maximum_absolute_error": 0.34375 + }, + { + "column": 89871, + "rms_error": 0.1417328324728651, + "maximum_absolute_error": 0.28125 + }, + { + "column": 63041, + "rms_error": 0.14120702039, + "maximum_absolute_error": 0.359375 + }, + { + "column": 1027, + "rms_error": 0.14041063075954496, + "maximum_absolute_error": 0.44921875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null + }, + "same_fp64_projected_rows_native_logits_sha256": "fb8dd6be86b662a95ffe5a04a94a4f6f6ccbf7164c019d3e0de023d742e90af8", + "same_fp64_projected_rows_native_diagnostic_sha256": "a6f6d5c4dfa719d9a2e90526a210bae07641da158c39b7772c200d0a1f17b468" + }, + { + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "vision": { + "relative_rms": 0.016063874280294436, + "maximum_absolute_error": 0.0087890625, + "equal_values": 16945, + "values": 143360, + "worst_columns": [ + { + "column": 498, + "rms": 0.0013091940942879653, + "max_abs": 0.0087890625 + }, + { + "column": 1764, + "rms": 0.001185183792016956, + "max_abs": 0.00732421875 + }, + { + "column": 809, + "rms": 0.0011589483218226793, + "max_abs": 0.00830078125 + }, + { + "column": 297, + "rms": 0.0011343321596087514, + "max_abs": 0.00714111328125 + }, + { + "column": 936, + "rms": 0.001095862149519839, + "max_abs": 0.00732421875 + }, + { + "column": 128, + "rms": 0.0010876887230548374, + "max_abs": 0.006103515625 + }, + { + "column": 652, + "rms": 0.0010590602510028337, + "max_abs": 0.005859375 + }, + { + "column": 1125, + "rms": 0.0010507872105445127, + "max_abs": 0.00604248046875 + }, + { + "column": 338, + "rms": 0.001043377334465457, + "max_abs": 0.00537109375 + }, + { + "column": 1960, + "rms": 0.0010385973306031866, + "max_abs": 0.00634765625 + }, + { + "column": 1608, + "rms": 0.0010359359747541973, + "max_abs": 0.0045166015625 + }, + { + "column": 1490, + "rms": 0.001009855121352609, + "max_abs": 0.00537109375 + }, + { + "column": 444, + "rms": 0.0010025326333805008, + "max_abs": 0.0068359375 + }, + { + "column": 397, + "rms": 0.0009856752557708178, + "max_abs": 0.00390625 + }, + { + "column": 1929, + "rms": 0.0009846070724547121, + "max_abs": 0.00390625 + }, + { + "column": 725, + "rms": 0.000984153556710703, + "max_abs": 0.00537109375 + }, + { + "column": 2022, + "rms": 0.0009788619755932626, + "max_abs": 0.0048828125 + }, + { + "column": 1777, + "rms": 0.000976186925169315, + "max_abs": 0.0048828125 + }, + { + "column": 1416, + "rms": 0.0009727661698073942, + "max_abs": 0.005615234375 + }, + { + "column": 1990, + "rms": 0.0009658278835947081, + "max_abs": 0.0057220458984375 + } + ], + "rows": [ + { + "row": 0, + "rms": 0.00038372740753499213, + "max_abs": 0.0013427734375 + }, + { + "row": 1, + "rms": 0.00038970468951474884, + "max_abs": 0.00146484375 + }, + { + "row": 2, + "rms": 0.0005385962238659673, + "max_abs": 0.00244140625 + }, + { + "row": 3, + "rms": 0.000532391804690437, + "max_abs": 0.001953125 + }, + { + "row": 4, + "rms": 0.00045059713743431107, + "max_abs": 0.001953125 + }, + { + "row": 5, + "rms": 0.000535412540394403, + "max_abs": 0.002197265625 + }, + { + "row": 6, + "rms": 0.0005446350280391706, + "max_abs": 0.001953125 + }, + { + "row": 7, + "rms": 0.00040900183839543433, + "max_abs": 0.001708984375 + }, + { + "row": 8, + "rms": 0.0005482143923531329, + "max_abs": 0.00244140625 + }, + { + "row": 9, + "rms": 0.0004320887828407833, + "max_abs": 0.0018310546875 + }, + { + "row": 10, + "rms": 0.0004493721978048441, + "max_abs": 0.001708984375 + }, + { + "row": 11, + "rms": 0.00048132567978884767, + "max_abs": 0.001953125 + }, + { + "row": 12, + "rms": 0.0004406089310737883, + "max_abs": 0.001735687255859375 + }, + { + "row": 13, + "rms": 0.00031486033651065233, + "max_abs": 0.00146484375 + }, + { + "row": 14, + "rms": 0.0006341479863918507, + "max_abs": 0.0029296875 + }, + { + "row": 15, + "rms": 0.0004002345030656259, + "max_abs": 0.00146484375 + }, + { + "row": 16, + "rms": 0.00046019922538941753, + "max_abs": 0.001953125 + }, + { + "row": 17, + "rms": 0.0004095935257686074, + "max_abs": 0.00146484375 + }, + { + "row": 18, + "rms": 0.0004174048032924595, + "max_abs": 0.001953125 + }, + { + "row": 19, + "rms": 0.0007863515636437885, + "max_abs": 0.00341796875 + }, + { + "row": 20, + "rms": 0.00046838318842117987, + "max_abs": 0.001953125 + }, + { + "row": 21, + "rms": 0.0004958759665262961, + "max_abs": 0.002197265625 + }, + { + "row": 22, + "rms": 0.0005967533694819817, + "max_abs": 0.00244140625 + }, + { + "row": 23, + "rms": 0.0004243509353734149, + "max_abs": 0.001708984375 + }, + { + "row": 24, + "rms": 0.0005357846031999338, + "max_abs": 0.0020599365234375 + }, + { + "row": 25, + "rms": 0.0004745263782153918, + "max_abs": 0.001708984375 + }, + { + "row": 26, + "rms": 0.0006049116600811196, + "max_abs": 0.00244140625 + }, + { + "row": 27, + "rms": 0.0012718290463919293, + "max_abs": 0.006103515625 + }, + { + "row": 28, + "rms": 0.0005543617429973317, + "max_abs": 0.00244140625 + }, + { + "row": 29, + "rms": 0.00046452820563499565, + "max_abs": 0.0020751953125 + }, + { + "row": 30, + "rms": 0.0004442953179747139, + "max_abs": 0.0019683837890625 + }, + { + "row": 31, + "rms": 0.00041361897628587705, + "max_abs": 0.001953125 + }, + { + "row": 32, + "rms": 0.0007000490247424031, + "max_abs": 0.0029296875 + }, + { + "row": 33, + "rms": 0.00047587498488368567, + "max_abs": 0.002197265625 + }, + { + "row": 34, + "rms": 0.00042651280049577295, + "max_abs": 0.00152587890625 + }, + { + "row": 35, + "rms": 0.0004061125662018534, + "max_abs": 0.0018310546875 + }, + { + "row": 36, + "rms": 0.0004777528468292542, + "max_abs": 0.001953125 + }, + { + "row": 37, + "rms": 0.0006343349672661181, + "max_abs": 0.0029296875 + }, + { + "row": 38, + "rms": 0.0004538810317093784, + "max_abs": 0.00146484375 + }, + { + "row": 39, + "rms": 0.0004983198958136023, + "max_abs": 0.001708984375 + }, + { + "row": 40, + "rms": 0.0005549457380957349, + "max_abs": 0.002197265625 + }, + { + "row": 41, + "rms": 0.000435093591768297, + "max_abs": 0.001953125 + }, + { + "row": 42, + "rms": 0.0016990532000404368, + "max_abs": 0.0087890625 + }, + { + "row": 43, + "rms": 0.0008343056505168412, + "max_abs": 0.00390625 + }, + { + "row": 44, + "rms": 0.0003863961261144731, + "max_abs": 0.0015869140625 + }, + { + "row": 45, + "rms": 0.0004130896401939031, + "max_abs": 0.001953125 + }, + { + "row": 46, + "rms": 0.0004656354475675182, + "max_abs": 0.001953125 + }, + { + "row": 47, + "rms": 0.00045687160490702494, + "max_abs": 0.001953125 + }, + { + "row": 48, + "rms": 0.0003940148584999622, + "max_abs": 0.001708984375 + }, + { + "row": 49, + "rms": 0.00042785081807080345, + "max_abs": 0.0015869140625 + }, + { + "row": 50, + "rms": 0.00042057304379851564, + "max_abs": 0.001708984375 + }, + { + "row": 51, + "rms": 0.0004346320529523064, + "max_abs": 0.00146484375 + }, + { + "row": 52, + "rms": 0.0007170450743457792, + "max_abs": 0.0040283203125 + }, + { + "row": 53, + "rms": 0.0005234081976803397, + "max_abs": 0.002197265625 + }, + { + "row": 54, + "rms": 0.0004300831551094984, + "max_abs": 0.001953125 + }, + { + "row": 55, + "rms": 0.00039691783537205294, + "max_abs": 0.001953125 + }, + { + "row": 56, + "rms": 0.0005015920315987764, + "max_abs": 0.002197265625 + }, + { + "row": 57, + "rms": 0.00047929273916284446, + "max_abs": 0.001953125 + }, + { + "row": 58, + "rms": 0.0005407952127570138, + "max_abs": 0.00225830078125 + }, + { + "row": 59, + "rms": 0.00042489098191947214, + "max_abs": 0.0015869140625 + }, + { + "row": 60, + "rms": 0.00039287798502104945, + "max_abs": 0.00146484375 + }, + { + "row": 61, + "rms": 0.0009851722974364375, + "max_abs": 0.003662109375 + }, + { + "row": 62, + "rms": 0.0005027295642356642, + "max_abs": 0.00209808349609375 + }, + { + "row": 63, + "rms": 0.0006288940981566789, + "max_abs": 0.002685546875 + }, + { + "row": 64, + "rms": 0.0006450404013174668, + "max_abs": 0.0029296875 + }, + { + "row": 65, + "rms": 0.00043091250461964635, + "max_abs": 0.001708984375 + }, + { + "row": 66, + "rms": 0.0005250592674429304, + "max_abs": 0.002197265625 + }, + { + "row": 67, + "rms": 0.000502115440446748, + "max_abs": 0.001953125 + }, + { + "row": 68, + "rms": 0.0006125769117026512, + "max_abs": 0.0029296875 + }, + { + "row": 69, + "rms": 0.00043968690926253284, + "max_abs": 0.001708984375 + } + ] + }, + "decoder": { + "rows": 90, + "vocabulary": 128000, + "top1_differences": 7, + "disjoint_maximum_sets": 7, + "relative_rms": 0.022216577726492024, + "maximum_absolute_error": 1.59375, + "differences": [ + { + "row": 14, + "native_top": 594, + "reference_top": 73, + "native_maximum_set": [ + 594 + ], + "reference_maximum_set": [ + 73 + ], + "maximum_sets_overlap": false, + "native_margin": 0.03125, + "reference_margin": 0.125, + "native_reference_contender_gap": 0.03125, + "reference_native_contender_gap": 0.125 + }, + { + "row": 16, + "native_top": 296, + "reference_top": 5434, + "native_maximum_set": [ + 296 + ], + "reference_maximum_set": [ + 5434 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.03125, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.03125 + }, + { + "row": 30, + "native_top": 442, + "reference_top": 933, + "native_maximum_set": [ + 442 + ], + "reference_maximum_set": [ + 933 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.375, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.375 + }, + { + "row": 43, + "native_top": 10, + "reference_top": 76, + "native_maximum_set": [ + 10 + ], + "reference_maximum_set": [ + 76 + ], + "maximum_sets_overlap": false, + "native_margin": 0.03125, + "reference_margin": 0.09375, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.1875 + }, + { + "row": 52, + "native_top": 933, + "reference_top": 1628, + "native_maximum_set": [ + 933 + ], + "reference_maximum_set": [ + 1628 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.125, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.125 + }, + { + "row": 81, + "native_top": 78, + "reference_top": 851, + "native_maximum_set": [ + 78 + ], + "reference_maximum_set": [ + 851 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.40625, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.40625 + }, + { + "row": 88, + "native_top": 309, + "reference_top": 22, + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0625 + } + ], + "worst_rows": [ + { + "row": 24, + "rms_error": 0.31325322278601186, + "maximum_absolute_error": 0.9375 + }, + { + "row": 31, + "rms_error": 0.2681649899244382, + "maximum_absolute_error": 1.59375 + }, + { + "row": 34, + "rms_error": 0.16841338748141546, + "maximum_absolute_error": 0.6171875 + }, + { + "row": 67, + "rms_error": 0.16210303542146617, + "maximum_absolute_error": 0.703125 + }, + { + "row": 33, + "rms_error": 0.15095154620912385, + "maximum_absolute_error": 0.625 + }, + { + "row": 53, + "rms_error": 0.15036858365991487, + "maximum_absolute_error": 0.57421875 + }, + { + "row": 30, + "rms_error": 0.13482398494303077, + "maximum_absolute_error": 0.75 + }, + { + "row": 17, + "rms_error": 0.12072373863358932, + "maximum_absolute_error": 0.859375 + }, + { + "row": 44, + "rms_error": 0.11925762957256145, + "maximum_absolute_error": 0.71875 + }, + { + "row": 14, + "rms_error": 0.11336928062644555, + "maximum_absolute_error": 0.53125 + }, + { + "row": 86, + "rms_error": 0.10675978399391804, + "maximum_absolute_error": 0.5 + }, + { + "row": 36, + "rms_error": 0.1053937939487098, + "maximum_absolute_error": 0.328125 + }, + { + "row": 41, + "rms_error": 0.10178887092355156, + "maximum_absolute_error": 0.48828125 + }, + { + "row": 64, + "rms_error": 0.09718724996465507, + "maximum_absolute_error": 0.3125 + }, + { + "row": 43, + "rms_error": 0.09638023980667045, + "maximum_absolute_error": 0.484375 + }, + { + "row": 68, + "rms_error": 0.0935305943283503, + "maximum_absolute_error": 0.3359375 + }, + { + "row": 39, + "rms_error": 0.09350297196104815, + "maximum_absolute_error": 0.53125 + }, + { + "row": 72, + "rms_error": 0.0931610655527511, + "maximum_absolute_error": 0.578125 + }, + { + "row": 78, + "rms_error": 0.09280685834883261, + "maximum_absolute_error": 0.3984375 + }, + { + "row": 19, + "rms_error": 0.0924378053660569, + "maximum_absolute_error": 0.390625 + } + ], + "worst_columns": [ + { + "column": 4834, + "rms_error": 0.18777707608441097, + "maximum_absolute_error": 1.515625 + }, + { + "column": 5190, + "rms_error": 0.18375509130325815, + "maximum_absolute_error": 1.59375 + }, + { + "column": 7427, + "rms_error": 0.17809605205210005, + "maximum_absolute_error": 1.33984375 + }, + { + "column": 1420, + "rms_error": 0.17780621065999735, + "maximum_absolute_error": 1.50390625 + }, + { + "column": 104982, + "rms_error": 0.17572533177244842, + "maximum_absolute_error": 1.361572265625 + }, + { + "column": 1427, + "rms_error": 0.17482033061070493, + "maximum_absolute_error": 1.390625 + }, + { + "column": 55707, + "rms_error": 0.17428537300900585, + "maximum_absolute_error": 1.3203125 + }, + { + "column": 1286, + "rms_error": 0.1736546483021194, + "maximum_absolute_error": 1.2578125 + }, + { + "column": 59610, + "rms_error": 0.1709961554280689, + "maximum_absolute_error": 1.3203125 + }, + { + "column": 20057, + "rms_error": 0.16892938459182663, + "maximum_absolute_error": 1.359375 + }, + { + "column": 93784, + "rms_error": 0.16866035906239255, + "maximum_absolute_error": 1.40625 + }, + { + "column": 15446, + "rms_error": 0.16754801296119753, + "maximum_absolute_error": 1.15625 + }, + { + "column": 4425, + "rms_error": 0.16690986018832024, + "maximum_absolute_error": 1.3046875 + }, + { + "column": 518, + "rms_error": 0.1664933463570474, + "maximum_absolute_error": 1.21484375 + }, + { + "column": 18728, + "rms_error": 0.1658665546261442, + "maximum_absolute_error": 1.265625 + }, + { + "column": 14998, + "rms_error": 0.16498297767301526, + "maximum_absolute_error": 1.34375 + }, + { + "column": 4538, + "rms_error": 0.16489613179356355, + "maximum_absolute_error": 1.140625 + }, + { + "column": 27942, + "rms_error": 0.16263808879588532, + "maximum_absolute_error": 1.296875 + }, + { + "column": 461, + "rms_error": 0.16203926337191119, + "maximum_absolute_error": 1.1953125 + }, + { + "column": 79410, + "rms_error": 0.16061915049310574, + "maximum_absolute_error": 1.09375 + } + ], + "accepted_threshold": null, + "fidelity_pass": null + }, + "text_suffix": { + "rows": 16, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 2, + "relative_rms": 0.013412548767931996, + "maximum_absolute_error": 0.5, + "differences": [ + { + "row": 81, + "native_top": 78, + "reference_top": 851, + "native_maximum_set": [ + 78 + ], + "reference_maximum_set": [ + 851 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.40625, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.40625 + }, + { + "row": 88, + "native_top": 309, + "reference_top": 22, + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0625 + } + ], + "worst_rows": [ + { + "row": 86, + "rms_error": 0.10675978399391804, + "maximum_absolute_error": 0.5 + }, + { + "row": 78, + "rms_error": 0.09280685834883261, + "maximum_absolute_error": 0.3984375 + }, + { + "row": 74, + "rms_error": 0.09105627293640264, + "maximum_absolute_error": 0.3125 + }, + { + "row": 81, + "rms_error": 0.08690320272162921, + "maximum_absolute_error": 0.46875 + }, + { + "row": 83, + "rms_error": 0.07422929154241324, + "maximum_absolute_error": 0.3125 + }, + { + "row": 82, + "rms_error": 0.06978134870027429, + "maximum_absolute_error": 0.25 + }, + { + "row": 87, + "rms_error": 0.06769017849777814, + "maximum_absolute_error": 0.3671875 + }, + { + "row": 76, + "rms_error": 0.06456068310215368, + "maximum_absolute_error": 0.296875 + }, + { + "row": 84, + "rms_error": 0.060482117213051155, + "maximum_absolute_error": 0.359375 + }, + { + "row": 89, + "rms_error": 0.05800107078466209, + "maximum_absolute_error": 0.375 + }, + { + "row": 85, + "rms_error": 0.05358876803081932, + "maximum_absolute_error": 0.265625 + }, + { + "row": 88, + "rms_error": 0.05151130755037016, + "maximum_absolute_error": 0.3056640625 + }, + { + "row": 80, + "rms_error": 0.0361086306259185, + "maximum_absolute_error": 0.1875 + }, + { + "row": 79, + "rms_error": 0.03443325156663833, + "maximum_absolute_error": 0.15625 + }, + { + "row": 77, + "rms_error": 0.03338368830443815, + "maximum_absolute_error": 0.1875 + }, + { + "row": 75, + "rms_error": 0.024779049568996354, + "maximum_absolute_error": 0.125 + } + ], + "worst_columns": [ + { + "column": 851, + "rms_error": 0.14803007642008176, + "maximum_absolute_error": 0.46875 + }, + { + "column": 39482, + "rms_error": 0.14585803594725463, + "maximum_absolute_error": 0.4375 + }, + { + "column": 267, + "rms_error": 0.14518959015660782, + "maximum_absolute_error": 0.5 + }, + { + "column": 2130, + "rms_error": 0.14279471889796103, + "maximum_absolute_error": 0.3203125 + }, + { + "column": 121589, + "rms_error": 0.14144994576953052, + "maximum_absolute_error": 0.375 + }, + { + "column": 2600, + "rms_error": 0.14098777601673493, + "maximum_absolute_error": 0.3203125 + }, + { + "column": 94304, + "rms_error": 0.13920726313441767, + "maximum_absolute_error": 0.375 + }, + { + "column": 335, + "rms_error": 0.138905181194094, + "maximum_absolute_error": 0.36181640625 + }, + { + "column": 36356, + "rms_error": 0.1388033391479404, + "maximum_absolute_error": 0.5 + }, + { + "column": 11325, + "rms_error": 0.13719926588137327, + "maximum_absolute_error": 0.29296875 + }, + { + "column": 2867, + "rms_error": 0.1368786557898186, + "maximum_absolute_error": 0.3955078125 + }, + { + "column": 23638, + "rms_error": 0.13651653048570972, + "maximum_absolute_error": 0.34375 + }, + { + "column": 39916, + "rms_error": 0.13632756536418084, + "maximum_absolute_error": 0.375 + }, + { + "column": 97766, + "rms_error": 0.13599136863694697, + "maximum_absolute_error": 0.375 + }, + { + "column": 6639, + "rms_error": 0.13471655366881852, + "maximum_absolute_error": 0.375 + }, + { + "column": 5657, + "rms_error": 0.1344608232421106, + "maximum_absolute_error": 0.33837890625 + }, + { + "column": 274, + "rms_error": 0.13350296058658054, + "maximum_absolute_error": 0.3125 + }, + { + "column": 105258, + "rms_error": 0.13335943001678674, + "maximum_absolute_error": 0.3125 + }, + { + "column": 2684, + "rms_error": 0.1323269213861456, + "maximum_absolute_error": 0.328125 + }, + { + "column": 23382, + "rms_error": 0.13196969487807017, + "maximum_absolute_error": 0.36328125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null + }, + "native_projected_rows_sha256": "aceb2b880177abfae718bdd660398b3cd7ee990d2dfcdce2a110d08dc4e33135", + "fp64_projected_rows_sha256": "bc435e6390349c8be4a8ff12102ab941036ea1b13f0a1b4b9f1222189dee97a3", + "native_logits_sha256": "db730509b6d7dcb3d6d134b59b6434dfb43ebc85cdd3fe351bc8cc4148b47642", + "fp64_logits_sha256": "c34a303517fe4ea85a1886f9bc9928febebd96b677c1c9412462c5967c8fbc42", + "native_forward_diagnostic_sha256": "e80a2a14c0d8ed20b62088d0d8b87be4beb5fed3eb1a7336b3b00f8ac5f74bee", + "native_vision_diagnostic_sha256": "8bffb39af3c87626457ff020f91a6b76d48d1b795dd918578ccca61cb2743292", + "fp64_vision_scope": "6-bit checkpoint evaluated directly; 8-bit shares all 138 vision/projector tensors byte-for-byte, verified named-tensor hash", + "same_fp64_projected_rows_decoder": { + "rows": 90, + "vocabulary": 128000, + "top1_differences": 3, + "disjoint_maximum_sets": 2, + "relative_rms": 0.014357626607166978, + "maximum_absolute_error": 0.546875, + "differences": [ + { + "row": 16, + "native_top": 296, + "reference_top": 5434, + "native_maximum_set": [ + 296 + ], + "reference_maximum_set": [ + 5434 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.03125, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.03125 + }, + { + "row": 43, + "native_top": 55648, + "reference_top": 76, + "native_maximum_set": [ + 55648 + ], + "reference_maximum_set": [ + 76 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.09375, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.09375 + }, + { + "row": 46, + "native_top": 23, + "reference_top": 25496, + "native_maximum_set": [ + 23, + 25496 + ], + "reference_maximum_set": [ + 25496 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + } + ], + "worst_rows": [ + { + "row": 24, + "rms_error": 0.1951356078840599, + "maximum_absolute_error": 0.546875 + }, + { + "row": 14, + "rms_error": 0.11464681088233711, + "maximum_absolute_error": 0.46875 + }, + { + "row": 78, + "rms_error": 0.1040942459980685, + "maximum_absolute_error": 0.427734375 + }, + { + "row": 64, + "rms_error": 0.09503784802495824, + "maximum_absolute_error": 0.375 + }, + { + "row": 87, + "rms_error": 0.08195156192731926, + "maximum_absolute_error": 0.28125 + }, + { + "row": 43, + "rms_error": 0.07636420302287246, + "maximum_absolute_error": 0.30078125 + }, + { + "row": 33, + "rms_error": 0.07558788651126319, + "maximum_absolute_error": 0.31640625 + }, + { + "row": 23, + "rms_error": 0.07478502289387448, + "maximum_absolute_error": 0.28125 + }, + { + "row": 86, + "rms_error": 0.07395806987605277, + "maximum_absolute_error": 0.375 + }, + { + "row": 3, + "rms_error": 0.07324018954373339, + "maximum_absolute_error": 0.4140625 + }, + { + "row": 44, + "rms_error": 0.07226988643114846, + "maximum_absolute_error": 0.40234375 + }, + { + "row": 83, + "rms_error": 0.06954534560799557, + "maximum_absolute_error": 0.3125 + }, + { + "row": 67, + "rms_error": 0.06890919882421954, + "maximum_absolute_error": 0.28125 + }, + { + "row": 34, + "rms_error": 0.06829569918962537, + "maximum_absolute_error": 0.37890625 + }, + { + "row": 2, + "rms_error": 0.0670275170659761, + "maximum_absolute_error": 0.3828125 + }, + { + "row": 84, + "rms_error": 0.06685557355776217, + "maximum_absolute_error": 0.3203125 + }, + { + "row": 74, + "rms_error": 0.06585552308303172, + "maximum_absolute_error": 0.28125 + }, + { + "row": 0, + "rms_error": 0.06483672638625593, + "maximum_absolute_error": 0.375 + }, + { + "row": 54, + "rms_error": 0.06445740549269832, + "maximum_absolute_error": 0.328125 + }, + { + "row": 21, + "rms_error": 0.0611484607565357, + "maximum_absolute_error": 0.21875 + } + ], + "worst_columns": [ + { + "column": 335, + "rms_error": 0.0958981073930199, + "maximum_absolute_error": 0.328125 + }, + { + "column": 295, + "rms_error": 0.0949124159283936, + "maximum_absolute_error": 0.3232421875 + }, + { + "column": 410, + "rms_error": 0.09242965586173113, + "maximum_absolute_error": 0.546875 + }, + { + "column": 2117, + "rms_error": 0.09061483658695522, + "maximum_absolute_error": 0.359375 + }, + { + "column": 66215, + "rms_error": 0.08976006134560217, + "maximum_absolute_error": 0.359375 + }, + { + "column": 90312, + "rms_error": 0.08853406686517258, + "maximum_absolute_error": 0.4375 + }, + { + "column": 3773, + "rms_error": 0.08806378296711033, + "maximum_absolute_error": 0.40234375 + }, + { + "column": 105532, + "rms_error": 0.08750445855036317, + "maximum_absolute_error": 0.5 + }, + { + "column": 3874, + "rms_error": 0.08723444971001719, + "maximum_absolute_error": 0.37890625 + }, + { + "column": 24975, + "rms_error": 0.08702265400405458, + "maximum_absolute_error": 0.5 + }, + { + "column": 47, + "rms_error": 0.08583225876866991, + "maximum_absolute_error": 0.28125 + }, + { + "column": 401, + "rms_error": 0.08574283785948696, + "maximum_absolute_error": 0.28125 + }, + { + "column": 311, + "rms_error": 0.08535110873665012, + "maximum_absolute_error": 0.248046875 + }, + { + "column": 293, + "rms_error": 0.08519964871844057, + "maximum_absolute_error": 0.2421875 + }, + { + "column": 959, + "rms_error": 0.08516147084228569, + "maximum_absolute_error": 0.25 + }, + { + "column": 117545, + "rms_error": 0.08515255732409178, + "maximum_absolute_error": 0.328125 + }, + { + "column": 1082, + "rms_error": 0.08514577507587652, + "maximum_absolute_error": 0.501953125 + }, + { + "column": 3031, + "rms_error": 0.08508267894397326, + "maximum_absolute_error": 0.390625 + }, + { + "column": 6596, + "rms_error": 0.08502455652136487, + "maximum_absolute_error": 0.3125 + }, + { + "column": 40714, + "rms_error": 0.08485773085712119, + "maximum_absolute_error": 0.46875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null + }, + "same_fp64_projected_rows_text_suffix": { + "rows": 16, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.012360610449916133, + "maximum_absolute_error": 0.427734375, + "differences": [], + "worst_rows": [ + { + "row": 78, + "rms_error": 0.1040942459980685, + "maximum_absolute_error": 0.427734375 + }, + { + "row": 87, + "rms_error": 0.08195156192731926, + "maximum_absolute_error": 0.28125 + }, + { + "row": 86, + "rms_error": 0.07395806987605277, + "maximum_absolute_error": 0.375 + }, + { + "row": 83, + "rms_error": 0.06954534560799557, + "maximum_absolute_error": 0.3125 + }, + { + "row": 84, + "rms_error": 0.06685557355776217, + "maximum_absolute_error": 0.3203125 + }, + { + "row": 74, + "rms_error": 0.06585552308303172, + "maximum_absolute_error": 0.28125 + }, + { + "row": 88, + "rms_error": 0.058481772557631394, + "maximum_absolute_error": 0.28125 + }, + { + "row": 89, + "rms_error": 0.05804265544486751, + "maximum_absolute_error": 0.375 + }, + { + "row": 82, + "rms_error": 0.05787463072332117, + "maximum_absolute_error": 0.21875 + }, + { + "row": 76, + "rms_error": 0.057330974169354156, + "maximum_absolute_error": 0.3984375 + }, + { + "row": 85, + "rms_error": 0.05509315111999264, + "maximum_absolute_error": 0.25 + }, + { + "row": 81, + "rms_error": 0.05234338783023947, + "maximum_absolute_error": 0.359375 + }, + { + "row": 80, + "rms_error": 0.04242200552065804, + "maximum_absolute_error": 0.1875 + }, + { + "row": 77, + "rms_error": 0.03785204590273343, + "maximum_absolute_error": 0.1875 + }, + { + "row": 75, + "rms_error": 0.031897078426270396, + "maximum_absolute_error": 0.125 + }, + { + "row": 79, + "rms_error": 0.028651777580885336, + "maximum_absolute_error": 0.15625 + } + ], + "worst_columns": [ + { + "column": 3773, + "rms_error": 0.14774971279690838, + "maximum_absolute_error": 0.40234375 + }, + { + "column": 9032, + "rms_error": 0.14462323741119032, + "maximum_absolute_error": 0.36474609375 + }, + { + "column": 10057, + "rms_error": 0.13571611332379835, + "maximum_absolute_error": 0.42578125 + }, + { + "column": 5140, + "rms_error": 0.1355807215402516, + "maximum_absolute_error": 0.3125 + }, + { + "column": 1475, + "rms_error": 0.1324892036070042, + "maximum_absolute_error": 0.30859375 + }, + { + "column": 3568, + "rms_error": 0.13087394974805433, + "maximum_absolute_error": 0.3984375 + }, + { + "column": 15121, + "rms_error": 0.13071080214349948, + "maximum_absolute_error": 0.2734375 + }, + { + "column": 17952, + "rms_error": 0.12857753572643024, + "maximum_absolute_error": 0.375 + }, + { + "column": 7504, + "rms_error": 0.12815034197026057, + "maximum_absolute_error": 0.375 + }, + { + "column": 30187, + "rms_error": 0.12686294874261703, + "maximum_absolute_error": 0.328125 + }, + { + "column": 4132, + "rms_error": 0.12648369490291603, + "maximum_absolute_error": 0.3984375 + }, + { + "column": 20317, + "rms_error": 0.12571891045423828, + "maximum_absolute_error": 0.34375 + }, + { + "column": 16566, + "rms_error": 0.12499332410151467, + "maximum_absolute_error": 0.2890625 + }, + { + "column": 3803, + "rms_error": 0.12480147809840585, + "maximum_absolute_error": 0.3984375 + }, + { + "column": 116016, + "rms_error": 0.12469158271090393, + "maximum_absolute_error": 0.333984375 + }, + { + "column": 12839, + "rms_error": 0.12433472563560184, + "maximum_absolute_error": 0.361572265625 + }, + { + "column": 1609, + "rms_error": 0.12318751456978148, + "maximum_absolute_error": 0.37109375 + }, + { + "column": 57364, + "rms_error": 0.12251870771532965, + "maximum_absolute_error": 0.3359375 + }, + { + "column": 1007, + "rms_error": 0.12189538247279294, + "maximum_absolute_error": 0.34375 + }, + { + "column": 2534, + "rms_error": 0.12184843116097803, + "maximum_absolute_error": 0.390625 + } + ], + "accepted_threshold": null, + "fidelity_pass": null + }, + "same_fp64_projected_rows_native_logits_sha256": "e88367024b33c1118da845a4df7011815ba7e94124dc8e4a5ce8dfa9d8b43598", + "same_fp64_projected_rows_native_diagnostic_sha256": "a6f6d5c4dfa719d9a2e90526a210bae07641da158c39b7772c200d0a1f17b468" + } + ] +} diff --git a/docs/receipts/lfm3/real-image-mlx-secondary.json b/docs/receipts/lfm3/real-image-mlx-secondary.json new file mode 100644 index 0000000000..fb88484482 --- /dev/null +++ b/docs/receipts/lfm3/real-image-mlx-secondary.json @@ -0,0 +1,68 @@ +{ + "scope": "Secondary MLX developer vision reference comparison on identical frozen native image patches, not official MLX-VLM end-to-end serving or a fidelity pass. 6/8 vision weights are identical by prior full-tensor hash verification; only distinct 4/8 sets are run here.", + "grid": [ + 14, + 20 + ], + "patches_sha256": "f1c55a0dd28b91d9929f074995adadea79d7c0862c8cdcd0be6dbfda57383dc1", + "host": "Apple M4 Pro, macOS 26.5.1, MLX 0.32.4", + "records": [ + { + "vision_weight_set": "4bit export", + "target": "Ideal FP64 vision oracle with its recorded BF16 stores; no acceptance threshold", + "comparators": { + "native": { + "output_sha256": "ab2ac7db759e5c4a32fe7d60680ec47f7d1cbc5938fa7d8091bf4756bfeea4fc", + "rms": 0.000641842947115359, + "relative_rms": 0.01813030697564529, + "max_abs": 0.009981155395507812, + "worst_column": 1990, + "worst_column_rms": 0.0015688593706404128, + "equal_bf16_values": 16235, + "values": 143360 + }, + "mlx_developer_reference": { + "output_sha256": "34a2b8adfb6b17949d16a7f6d0762d1196c8151c5fde19a2bcb93d83c4f84f83", + "rms": 0.0005991838235835683, + "relative_rms": 0.016925303464397964, + "max_abs": 0.00634765625, + "worst_column": 397, + "worst_column_rms": 0.0012908817021458044, + "equal_bf16_values": 16157, + "values": 143360 + } + } + }, + { + "vision_weight_set": "8bit export", + "target": "Ideal FP64 vision oracle with its recorded BF16 stores; no acceptance threshold", + "comparators": { + "native": { + "output_sha256": "aceb2b880177abfae718bdd660398b3cd7ee990d2dfcdce2a110d08dc4e33135", + "rms": 0.0005656584034634817, + "relative_rms": 0.016063874280294436, + "max_abs": 0.0087890625, + "worst_column": 498, + "worst_column_rms": 0.0013091940942879653, + "equal_bf16_values": 16945, + "values": 143360 + }, + "mlx_developer_reference": { + "output_sha256": "4eb3954a4b7c0d5d8e31d01ca2393ef754dbcc1605ad5c71b6679f3b7bbe6039", + "rms": 0.0005638638806455016, + "relative_rms": 0.016012912447561735, + "max_abs": 0.0078125, + "worst_column": 128, + "worst_column_rms": 0.0012127937651889588, + "equal_bf16_values": 17083, + "values": 143360 + } + } + } + ], + "source_sha256": { + "tools/lfm3/vision_reference.py": "99ec81e9ff78e58f9e75c5562466b87aaf520a9a2c2da9a45a0593e587d7b711", + "tools/lfm3/reference.py": "fd4622a6fc35ed61ba4bf7a9d48c06a43e7cc390bc6505e7e6445300386b9f39" + }, + "command": "python tools/lfm3/vision_reference.py --model-dir MODEL --patches PATCHES.npy --grid-h 14 --grid-w 20 --output OUTPUT.bf16" +} diff --git a/docs/receipts/lfm3/real-image-rounding-reproduction.json b/docs/receipts/lfm3/real-image-rounding-reproduction.json new file mode 100644 index 0000000000..25c8ab6980 --- /dev/null +++ b/docs/receipts/lfm3/real-image-rounding-reproduction.json @@ -0,0 +1,349 @@ +{ + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "phase": "prompt", + "image": { + "url": "https://placecats.com/300/200", + "jpeg_sha256": "d1b324f529a96d8d8a16d973d88e1e974905d9c2c0e3e8b21a50b1c0e3fe2379", + "canonical_png_sha256": "005500792ee8ea6f35ac369d900fc788cd77667f145e7100c2cff57fae5355d9", + "conversion": "macOS sips JPEG to PNG; canonical PNG is the exact native/reference input" + }, + "grid": [ + 14, + 20 + ], + "image_start": 4, + "image_rows": 70, + "suffix_start": 74, + "suffix_rows": 16, + "scope": "Independent CPU FP32 arithmetic replay with explicitly delegated Metal scalar elementary functions; not an independent FP64 pass", + "ideal_fp64_retained": { + "receipt": "real-image-fp64.json", + "suffix_differences": [ + { + "row": 81, + "native_top": 78, + "reference_top": 851, + "native_maximum_set": [ + 78 + ], + "reference_maximum_set": [ + 851 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.40625, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.40625 + }, + { + "row": 88, + "native_top": 309, + "reference_top": 22, + "native_maximum_set": [ + 309 + ], + "reference_maximum_set": [ + 22 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0625 + } + ] + }, + "vision_stages": [ + { + "stage": "patchifier", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "embedding_norm", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_00", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_01", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_02", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_03", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_04", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_05", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_06", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_07", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_08", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_09", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_10", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_11", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_12", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "layer_13", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "projector_norm", + "values": 286720, + "different_bf16": 0 + }, + { + "stage": "projector_gelu", + "values": 286720, + "different_bf16": 0 + } + ], + "text_stages": [ + { + "stage": "layer_00_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_01_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_02_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_03_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_04_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_05_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_06_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_07_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_08_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_09_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_10_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_11_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_12_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_13_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_14_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_15_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_16_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_17_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_18_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_19_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_20_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_21_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_22_output", + "values": 184320, + "different_bf16": 0 + }, + { + "stage": "layer_23_output", + "values": 184320, + "different_bf16": 0 + } + ], + "projected_image": { + "values": 143360, + "different_bf16": 0, + "sha256": "aceb2b880177abfae718bdd660398b3cd7ee990d2dfcdce2a110d08dc4e33135" + }, + "decoder_logits": { + "values": 11520000, + "different_bf16": 0, + "sha256": "db730509b6d7dcb3d6d134b59b6434dfb43ebc85cdd3fe351bc8cc4148b47642", + "suffix_top1_differences": 0 + }, + "first_attention_elementary_diagnostic": { + "scope": "Fixed native Q/K/V, independently replayed FP32 arithmetic with captured elementary functions", + "rows": 280, + "score_values": 1254400, + "score_differences": 0, + "raw_values": 291200, + "raw_differences": 0, + "helper_context_differences": 0, + "exp_values": 2508800, + "exp_differences_vs_fp64_rounded": 463691, + "maximum_exp_ulp_vs_fp64_rounded": 1, + "accepted_threshold": null, + "whole_model_fidelity_pass": null + }, + "rounding_causes": [ + "FP32 lane reduction/FMA sites", + "FP32 normalization reduction and rsqrt sites", + "Metal exp/exp2/cos/sin elementary rounding", + "FP32 subnormal normalization products flushed before BF16 store" + ], + "rotary_fixed_primitive_replay": { + "query_values": 286720, + "key_values": 143360, + "different_bf16": 0 + }, + "accepted_threshold": null, + "independent_fp64_fidelity_pass": null, + "runtime_changes": "Trace-only snapshots; no arithmetic changes", + "reproduction": "fidelity-replay.md#real-image-rounding-causality", + "artifact_identity": { + "tested_host": { + "chip": "Apple M4 Pro", + "model_identifier": "Mac16,7", + "memory_gib": 48, + "macos": "26.5.1", + "macos_build": "25F80", + "architecture": "arm64" + }, + "native_decoder_baseline": { + "diagnostic_sha256": "e80a2a14c0d8ed20b62088d0d8b87be4beb5fed3eb1a7336b3b00f8ac5f74bee", + "logits_sha256": "db730509b6d7dcb3d6d134b59b6434dfb43ebc85cdd3fe351bc8cc4148b47642", + "scope": "Previously frozen native baseline from real-image-fp64.json; not a claim that later runtime revisions were requalified" + }, + "native_vision_canonical_recapture": { + "diagnostic_sha256": "7a635c8439842ae2bf8e7993cacf1a8cea0a386f57033654d74c08e65292efea", + "projected_image_sha256": "aceb2b880177abfae718bdd660398b3cd7ee990d2dfcdce2a110d08dc4e33135", + "source_sha256": { + "zig/src/families/lfm3/vision.zig": "c80661799c9e9fd7d71d6cbb2838c9701388cd3f292496f346956b3edd8a21e0", + "zig/kernels/metal/lfm3/vision.metal": "7f6338698948bebe80fe21b60b2c68c0a55a08208e8a768715016a02976e846a" + } + }, + "reference_source_sha256": { + "tools/lfm3/fp64_reference.py": "0fb1128ade095db87b6126792ab14930792ab1f7b41b998570356bc586f16b2e", + "tools/lfm3/row_projection_reference.py": "d03f224a037a920ba66b9828059d1489e48729cd168077e76789d0ec5c199c0c", + "tools/lfm3/row_projection_replay.c": "ff336f75fcf0a4f6c0ac92ed627bc1a6abb0b36b8f4cc95d876f2a8bb2f5cc42", + "tools/lfm3/elementary_reference.py": "3e4db276278867b7683536044ed7bcbe4c7643867d722fdda5b8ad895b3963e7", + "tools/lfm3/text_rounding_reference.py": "52665eef48570d45a2554861fe8f22b31874f8f464366adf882f40c3d311db27", + "tools/lfm3/text_attention_rounding.py": "ae67f2b2c26f89f75b1f36822a4b30a9f710df2b42f9f7ec42679373028b89e8", + "tools/lfm3/vision_fp64_reference.py": "742b8b4f44dde824c3a5c2d72a632401a0f9df7b7f23a8d3729d15f4f8c6a4d2", + "tools/lfm3/vision_rounding_reference.py": "8b2f5679bbd3f05b3e86a5748c3803f744502cffd0159bee2a218da38bd6f1c4", + "tools/lfm3/vision_attention_rounding.py": "37d99908f7b324740d098276f3d014cc9cc740587a8b798cde8be83d6dcef60d" + }, + "optional_c_accelerator_sha256": "fd3bdcc07800ca3e4bda472228c5fc86afa1f8eab7c2ba4669b0eae1e07df82b", + "elementary_capture": { + "shader_sha256": "7f6338698948bebe80fe21b60b2c68c0a55a08208e8a768715016a02976e846a", + "scope": "Actual production helper diagnostic, not independent FP64 reference" + } + } +} diff --git a/docs/receipts/lfm3/rounded-fp64-prompt.json b/docs/receipts/lfm3/rounded-fp64-prompt.json new file mode 100644 index 0000000000..901fe5641b --- /dev/null +++ b/docs/receipts/lfm3/rounded-fp64-prompt.json @@ -0,0 +1,5054 @@ +{ + "scope": "Actual FP32 affine projection reduction replay in independent FP64 model; remaining nonprojection operations retain FP64 arithmetic", + "accepted_threshold": null, + "fidelity_pass": null, + "completed_cells": 17, + "requested_cells": 17, + "cells": [ + { + "rows": 257, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.017635787525519895, + "maximum_absolute_error": 1.125, + "differences": [ + { + "row": 58, + "native_top": 3599, + "reference_top": 2292, + "native_maximum_set": [ + 3599 + ], + "reference_maximum_set": [ + 2292, + 3599 + ], + "maximum_sets_overlap": true, + "native_margin": 0.375, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.375, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 202, + "rms_error": 0.257782799033294, + "maximum_absolute_error": 0.9453125 + }, + { + "row": 250, + "rms_error": 0.18972153068496678, + "maximum_absolute_error": 1.0 + }, + { + "row": 106, + "rms_error": 0.18239645367221774, + "maximum_absolute_error": 1.125 + }, + { + "row": 82, + "rms_error": 0.1425757461819509, + "maximum_absolute_error": 0.7578125 + }, + { + "row": 226, + "rms_error": 0.11753441773676597, + "maximum_absolute_error": 0.5078125 + }, + { + "row": 137, + "rms_error": 0.1153958135874661, + "maximum_absolute_error": 0.59375 + }, + { + "row": 154, + "rms_error": 0.09746047597567065, + "maximum_absolute_error": 0.53125 + }, + { + "row": 118, + "rms_error": 0.09515304941526567, + "maximum_absolute_error": 0.44140625 + }, + { + "row": 178, + "rms_error": 0.09122065242154616, + "maximum_absolute_error": 0.4609375 + }, + { + "row": 161, + "rms_error": 0.08692003525411107, + "maximum_absolute_error": 0.40625 + }, + { + "row": 113, + "rms_error": 0.07981287396392105, + "maximum_absolute_error": 0.296875 + }, + { + "row": 209, + "rms_error": 0.07628492176773138, + "maximum_absolute_error": 0.4267578125 + }, + { + "row": 130, + "rms_error": 0.07466033800046858, + "maximum_absolute_error": 0.4140625 + }, + { + "row": 46, + "rms_error": 0.07419616631932618, + "maximum_absolute_error": 0.296875 + }, + { + "row": 254, + "rms_error": 0.07285502848213199, + "maximum_absolute_error": 0.375 + }, + { + "row": 142, + "rms_error": 0.07284207230221178, + "maximum_absolute_error": 0.3125 + }, + { + "row": 253, + "rms_error": 0.07159130505650123, + "maximum_absolute_error": 0.234375 + }, + { + "row": 181, + "rms_error": 0.06925787568477583, + "maximum_absolute_error": 0.234375 + }, + { + "row": 229, + "rms_error": 0.06869786843880964, + "maximum_absolute_error": 0.21875 + }, + { + "row": 77, + "rms_error": 0.06715550854939922, + "maximum_absolute_error": 0.3125 + } + ], + "worst_columns": [ + { + "column": 38520, + "rms_error": 0.11248981312110265, + "maximum_absolute_error": 1.0 + }, + { + "column": 90344, + "rms_error": 0.11157782703431807, + "maximum_absolute_error": 1.0 + }, + { + "column": 3008, + "rms_error": 0.10686136779291906, + "maximum_absolute_error": 1.0 + }, + { + "column": 2789, + "rms_error": 0.10605007369098719, + "maximum_absolute_error": 1.125 + }, + { + "column": 35070, + "rms_error": 0.10400304265142912, + "maximum_absolute_error": 0.921875 + }, + { + "column": 26668, + "rms_error": 0.10354200133120424, + "maximum_absolute_error": 0.8203125 + }, + { + "column": 52902, + "rms_error": 0.10323904671549591, + "maximum_absolute_error": 0.78125 + }, + { + "column": 77351, + "rms_error": 0.10180908512858072, + "maximum_absolute_error": 0.921875 + }, + { + "column": 1286, + "rms_error": 0.10126399199587542, + "maximum_absolute_error": 0.9375 + }, + { + "column": 63660, + "rms_error": 0.10017381513471851, + "maximum_absolute_error": 0.84375 + }, + { + "column": 2419, + "rms_error": 0.10012487530794274, + "maximum_absolute_error": 0.875 + }, + { + "column": 69072, + "rms_error": 0.09975898035670497, + "maximum_absolute_error": 0.8984375 + }, + { + "column": 23385, + "rms_error": 0.09950673886813924, + "maximum_absolute_error": 0.8046875 + }, + { + "column": 30687, + "rms_error": 0.09929726233056745, + "maximum_absolute_error": 0.6875 + }, + { + "column": 682, + "rms_error": 0.09903989271669766, + "maximum_absolute_error": 1.03125 + }, + { + "column": 22501, + "rms_error": 0.09861036577427301, + "maximum_absolute_error": 0.78125 + }, + { + "column": 2292, + "rms_error": 0.09823282868595429, + "maximum_absolute_error": 0.875 + }, + { + "column": 38173, + "rms_error": 0.0980188528167758, + "maximum_absolute_error": 0.84375 + }, + { + "column": 18731, + "rms_error": 0.09787894310090144, + "maximum_absolute_error": 0.75 + }, + { + "column": 118692, + "rms_error": 0.09592147692708017, + "maximum_absolute_error": 0.921875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "1.7B-6-prose-257", + "phase": "prompt", + "seconds": 127.02277824981138, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "2fb12160d203b843337c07b5a3e5b7a1f35de658857c3e06d3827c77dac1d001", + "rounded_reference_logits_sha256": "7b94f202ddc70106c25dcc260f29466e6bfea00ba29b61273603d26b87663c15", + "token_fixture_sha256": "522ec0f7a7ef8481d2f5533e3962ddad1f418d4337b095282e40c096e2befb1d", + "original_ideal_fp64_differences": [ + { + "row": 58, + "native_top": 3599, + "reference_top": 2292, + "native_maximum_set": [ + 3599 + ], + "reference_maximum_set": [ + 2292, + 3599 + ], + "maximum_sets_overlap": true, + "native_margin": 0.375, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.375, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 2292, + "rounded_reference_margin": 0.0, + "rounded_reference_native_contender_gap": 0.0, + "rounded_reference_maximum_set": [ + 2292, + 3599 + ] + } + ] + }, + { + "rows": 481, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.0210872789742184, + "maximum_absolute_error": 2.53125, + "differences": [ + { + "row": 58, + "native_top": 3599, + "reference_top": 2292, + "native_maximum_set": [ + 3599 + ], + "reference_maximum_set": [ + 2292, + 3599 + ], + "maximum_sets_overlap": true, + "native_margin": 0.375, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.375, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 353, + "rms_error": 0.4063506040364455, + "maximum_absolute_error": 2.125 + }, + { + "row": 329, + "rms_error": 0.3718739029875104, + "maximum_absolute_error": 1.46875 + }, + { + "row": 425, + "rms_error": 0.3697700436864613, + "maximum_absolute_error": 2.53125 + }, + { + "row": 449, + "rms_error": 0.2716952824894134, + "maximum_absolute_error": 1.6875 + }, + { + "row": 322, + "rms_error": 0.26884989379432517, + "maximum_absolute_error": 1.078125 + }, + { + "row": 202, + "rms_error": 0.257782799033294, + "maximum_absolute_error": 0.9453125 + }, + { + "row": 401, + "rms_error": 0.24028185259431445, + "maximum_absolute_error": 1.5625 + }, + { + "row": 377, + "rms_error": 0.2094349605874789, + "maximum_absolute_error": 1.3125 + }, + { + "row": 250, + "rms_error": 0.18972153068496678, + "maximum_absolute_error": 1.0 + }, + { + "row": 473, + "rms_error": 0.1867577649623245, + "maximum_absolute_error": 1.5 + }, + { + "row": 106, + "rms_error": 0.18239645367221774, + "maximum_absolute_error": 1.125 + }, + { + "row": 82, + "rms_error": 0.1425757461819509, + "maximum_absolute_error": 0.7578125 + }, + { + "row": 298, + "rms_error": 0.1414657185753844, + "maximum_absolute_error": 0.828125 + }, + { + "row": 226, + "rms_error": 0.11753441773676597, + "maximum_absolute_error": 0.5078125 + }, + { + "row": 137, + "rms_error": 0.1153958135874661, + "maximum_absolute_error": 0.59375 + }, + { + "row": 394, + "rms_error": 0.1012730697653547, + "maximum_absolute_error": 0.5 + }, + { + "row": 154, + "rms_error": 0.09746047597567065, + "maximum_absolute_error": 0.53125 + }, + { + "row": 118, + "rms_error": 0.09515304941526567, + "maximum_absolute_error": 0.44140625 + }, + { + "row": 178, + "rms_error": 0.09122065242154616, + "maximum_absolute_error": 0.4609375 + }, + { + "row": 161, + "rms_error": 0.08692003525411107, + "maximum_absolute_error": 0.40625 + } + ], + "worst_columns": [ + { + "column": 42654, + "rms_error": 0.22104044060462807, + "maximum_absolute_error": 2.53125 + }, + { + "column": 17944, + "rms_error": 0.2194537635352515, + "maximum_absolute_error": 2.5 + }, + { + "column": 117811, + "rms_error": 0.20100304282374698, + "maximum_absolute_error": 2.2265625 + }, + { + "column": 10601, + "rms_error": 0.20066892064495043, + "maximum_absolute_error": 2.375 + }, + { + "column": 44048, + "rms_error": 0.19217726886860778, + "maximum_absolute_error": 2.1875 + }, + { + "column": 40883, + "rms_error": 0.18727124669886536, + "maximum_absolute_error": 2.125 + }, + { + "column": 53632, + "rms_error": 0.18718837165693786, + "maximum_absolute_error": 2.109375 + }, + { + "column": 50974, + "rms_error": 0.18532977361824887, + "maximum_absolute_error": 2.03125 + }, + { + "column": 96351, + "rms_error": 0.18296954801725443, + "maximum_absolute_error": 2.0 + }, + { + "column": 31261, + "rms_error": 0.17384329629769307, + "maximum_absolute_error": 1.78125 + }, + { + "column": 3773, + "rms_error": 0.17248500613895462, + "maximum_absolute_error": 1.785888671875 + }, + { + "column": 39431, + "rms_error": 0.16968463580189658, + "maximum_absolute_error": 1.8515625 + }, + { + "column": 47454, + "rms_error": 0.1663327327036182, + "maximum_absolute_error": 1.97265625 + }, + { + "column": 99117, + "rms_error": 0.16469619804757268, + "maximum_absolute_error": 1.845703125 + }, + { + "column": 18961, + "rms_error": 0.1629467717748216, + "maximum_absolute_error": 1.6875 + }, + { + "column": 49422, + "rms_error": 0.16067112433148215, + "maximum_absolute_error": 1.640625 + }, + { + "column": 63437, + "rms_error": 0.1606512406804819, + "maximum_absolute_error": 1.875 + }, + { + "column": 33454, + "rms_error": 0.1588589496929098, + "maximum_absolute_error": 1.640625 + }, + { + "column": 48248, + "rms_error": 0.15858018914911048, + "maximum_absolute_error": 1.98046875 + }, + { + "column": 19982, + "rms_error": 0.15688888747079888, + "maximum_absolute_error": 1.728515625 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "1.7B-6-prose-513", + "phase": "prompt", + "seconds": 233.82947916723788, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "b6c07fed3bdccf311a1a46b1b2dc3a5f3de069d0b0a433d679a25b41cfa15968", + "rounded_reference_logits_sha256": "2a0d494528a8b0aee3249fe63f2281d9d3758f07dc24083144f7bbe7d5fd5620", + "token_fixture_sha256": "4b9e533cc1948ee72c4231178ec3af21808d7542c97f35997bb9538161d9a4cb", + "original_ideal_fp64_differences": [ + { + "row": 58, + "native_top": 3599, + "reference_top": 2292, + "native_maximum_set": [ + 3599 + ], + "reference_maximum_set": [ + 2292, + 3599 + ], + "maximum_sets_overlap": true, + "native_margin": 0.375, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.375, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 2292, + "rounded_reference_margin": 0.0, + "rounded_reference_native_contender_gap": 0.0, + "rounded_reference_maximum_set": [ + 2292, + 3599 + ] + } + ] + }, + { + "rows": 513, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.02058228707473938, + "maximum_absolute_error": 3.6796875, + "differences": [ + { + "row": 450, + "native_top": 22497, + "reference_top": 360, + "native_maximum_set": [ + 22497 + ], + "reference_maximum_set": [ + 360, + 22497 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 284, + "rms_error": 0.5887454519157178, + "maximum_absolute_error": 3.6796875 + }, + { + "row": 407, + "rms_error": 0.5037606466443229, + "maximum_absolute_error": 2.883544921875 + }, + { + "row": 325, + "rms_error": 0.4096832865188425, + "maximum_absolute_error": 2.96875 + }, + { + "row": 489, + "rms_error": 0.22528932318472958, + "maximum_absolute_error": 1.1875 + }, + { + "row": 366, + "rms_error": 0.20536768716715706, + "maximum_absolute_error": 1.78125 + }, + { + "row": 243, + "rms_error": 0.19155319230632445, + "maximum_absolute_error": 1.05859375 + }, + { + "row": 202, + "rms_error": 0.14457214257057088, + "maximum_absolute_error": 0.8125 + }, + { + "row": 145, + "rms_error": 0.11963659947740767, + "maximum_absolute_error": 0.4375 + }, + { + "row": 448, + "rms_error": 0.11852102532024672, + "maximum_absolute_error": 0.71875 + }, + { + "row": 144, + "rms_error": 0.11464445671983226, + "maximum_absolute_error": 0.5625 + }, + { + "row": 120, + "rms_error": 0.1130358818991185, + "maximum_absolute_error": 0.6201171875 + }, + { + "row": 346, + "rms_error": 0.10790701371447747, + "maximum_absolute_error": 0.484375 + }, + { + "row": 41, + "rms_error": 0.10725328432409853, + "maximum_absolute_error": 0.41015625 + }, + { + "row": 185, + "rms_error": 0.10204863410092394, + "maximum_absolute_error": 0.6875 + }, + { + "row": 285, + "rms_error": 0.0993359959099533, + "maximum_absolute_error": 0.65625 + }, + { + "row": 412, + "rms_error": 0.09557567429988036, + "maximum_absolute_error": 0.515625 + }, + { + "row": 28, + "rms_error": 0.0891926572001992, + "maximum_absolute_error": 0.4375 + }, + { + "row": 377, + "rms_error": 0.08871935864233524, + "maximum_absolute_error": 0.3125 + }, + { + "row": 279, + "rms_error": 0.08812270319058221, + "maximum_absolute_error": 0.34375 + }, + { + "row": 472, + "rms_error": 0.08434257945790119, + "maximum_absolute_error": 0.381591796875 + } + ], + "worst_columns": [ + { + "column": 33237, + "rms_error": 0.23781521468197503, + "maximum_absolute_error": 3.6796875 + }, + { + "column": 61089, + "rms_error": 0.22825897459601302, + "maximum_absolute_error": 3.3359375 + }, + { + "column": 16033, + "rms_error": 0.1985219891014985, + "maximum_absolute_error": 3.01171875 + }, + { + "column": 2320, + "rms_error": 0.19302383095424144, + "maximum_absolute_error": 3.41015625 + }, + { + "column": 116016, + "rms_error": 0.19245349326112937, + "maximum_absolute_error": 2.8203125 + }, + { + "column": 39382, + "rms_error": 0.1843450126790701, + "maximum_absolute_error": 2.6015625 + }, + { + "column": 45707, + "rms_error": 0.18066543235719704, + "maximum_absolute_error": 2.75 + }, + { + "column": 22690, + "rms_error": 0.17341389251236847, + "maximum_absolute_error": 2.5 + }, + { + "column": 93467, + "rms_error": 0.17203950724101422, + "maximum_absolute_error": 3.1015625 + }, + { + "column": 3632, + "rms_error": 0.1712014229043949, + "maximum_absolute_error": 3.0419921875 + }, + { + "column": 30270, + "rms_error": 0.1701286214424929, + "maximum_absolute_error": 2.34716796875 + }, + { + "column": 20921, + "rms_error": 0.17001152818826273, + "maximum_absolute_error": 3.04296875 + }, + { + "column": 12325, + "rms_error": 0.16956171987065036, + "maximum_absolute_error": 3.009765625 + }, + { + "column": 10292, + "rms_error": 0.16910886033329076, + "maximum_absolute_error": 2.6015625 + }, + { + "column": 8830, + "rms_error": 0.16776922840502032, + "maximum_absolute_error": 3.296875 + }, + { + "column": 6804, + "rms_error": 0.166636637947747, + "maximum_absolute_error": 2.765625 + }, + { + "column": 26176, + "rms_error": 0.16606035276026024, + "maximum_absolute_error": 2.8671875 + }, + { + "column": 10390, + "rms_error": 0.16588812937413056, + "maximum_absolute_error": 2.21875 + }, + { + "column": 5272, + "rms_error": 0.16577828256725613, + "maximum_absolute_error": 2.19140625 + }, + { + "column": 15243, + "rms_error": 0.1655292790273717, + "maximum_absolute_error": 2.64453125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "1.7B-8-code-513", + "phase": "prompt", + "seconds": 194.17714525014162, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "42aff32c97dbb25fabbd1415dbbb4b8f85b142f72fbc63b9a6666637a7047bb1", + "rounded_reference_logits_sha256": "8a248c50420876ee65a36c267aeef28913353b380ba11fe826596a946a60a51b", + "token_fixture_sha256": "c932ae46fd83f6e834a4001bd27496af3ebf54ac841a694d03a64558ba0daacb", + "original_ideal_fp64_differences": [] + }, + { + "rows": 513, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.012320885466310908, + "maximum_absolute_error": 0.61328125, + "differences": [ + { + "row": 9, + "native_top": 22, + "reference_top": 1046, + "native_maximum_set": [ + 22, + 1046 + ], + "reference_maximum_set": [ + 1046 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + } + ], + "worst_rows": [ + { + "row": 243, + "rms_error": 0.1357526025111896, + "maximum_absolute_error": 0.46875 + }, + { + "row": 79, + "rms_error": 0.11985875825420919, + "maximum_absolute_error": 0.34375 + }, + { + "row": 284, + "rms_error": 0.1179611715232355, + "maximum_absolute_error": 0.61328125 + }, + { + "row": 100, + "rms_error": 0.11074837554627359, + "maximum_absolute_error": 0.3427734375 + }, + { + "row": 54, + "rms_error": 0.10064829932890783, + "maximum_absolute_error": 0.267578125 + }, + { + "row": 69, + "rms_error": 0.09267317755762895, + "maximum_absolute_error": 0.3046875 + }, + { + "row": 120, + "rms_error": 0.07892488690670037, + "maximum_absolute_error": 0.28125 + }, + { + "row": 448, + "rms_error": 0.07616523302192119, + "maximum_absolute_error": 0.46875 + }, + { + "row": 362, + "rms_error": 0.07579456381253731, + "maximum_absolute_error": 0.1875 + }, + { + "row": 13, + "rms_error": 0.07550279825744916, + "maximum_absolute_error": 0.25 + }, + { + "row": 198, + "rms_error": 0.07398066242799343, + "maximum_absolute_error": 0.1875 + }, + { + "row": 480, + "rms_error": 0.06791453456599528, + "maximum_absolute_error": 0.16015625 + }, + { + "row": 167, + "rms_error": 0.06626552224756059, + "maximum_absolute_error": 0.265625 + }, + { + "row": 18, + "rms_error": 0.06518515816522075, + "maximum_absolute_error": 0.3125 + }, + { + "row": 29, + "rms_error": 0.06502488800863757, + "maximum_absolute_error": 0.21875 + }, + { + "row": 95, + "rms_error": 0.06375305225056092, + "maximum_absolute_error": 0.40625 + }, + { + "row": 92, + "rms_error": 0.06299632252398911, + "maximum_absolute_error": 0.21875 + }, + { + "row": 374, + "rms_error": 0.06267673788537, + "maximum_absolute_error": 0.21875 + }, + { + "row": 450, + "rms_error": 0.061246631092083545, + "maximum_absolute_error": 0.21875 + }, + { + "row": 16, + "rms_error": 0.06111191913120834, + "maximum_absolute_error": 0.203125 + } + ], + "worst_columns": [ + { + "column": 33237, + "rms_error": 0.07820019013719434, + "maximum_absolute_error": 0.37890625 + }, + { + "column": 20975, + "rms_error": 0.07754121379544646, + "maximum_absolute_error": 0.46875 + }, + { + "column": 16033, + "rms_error": 0.07201004797077867, + "maximum_absolute_error": 0.2578125 + }, + { + "column": 22497, + "rms_error": 0.07165507531127681, + "maximum_absolute_error": 0.3828125 + }, + { + "column": 116016, + "rms_error": 0.06885313731559682, + "maximum_absolute_error": 0.265625 + }, + { + "column": 15960, + "rms_error": 0.06643098034531959, + "maximum_absolute_error": 0.375 + }, + { + "column": 22657, + "rms_error": 0.06570281892092233, + "maximum_absolute_error": 0.375 + }, + { + "column": 5272, + "rms_error": 0.06509443690001435, + "maximum_absolute_error": 0.296875 + }, + { + "column": 62977, + "rms_error": 0.06484690101532765, + "maximum_absolute_error": 0.3125 + }, + { + "column": 61089, + "rms_error": 0.06444478085618802, + "maximum_absolute_error": 0.296875 + }, + { + "column": 42847, + "rms_error": 0.06389404967943242, + "maximum_absolute_error": 0.28125 + }, + { + "column": 8, + "rms_error": 0.06323179220435464, + "maximum_absolute_error": 0.25 + }, + { + "column": 106633, + "rms_error": 0.06304496069875547, + "maximum_absolute_error": 0.28125 + }, + { + "column": 495, + "rms_error": 0.06275284709399459, + "maximum_absolute_error": 0.328125 + }, + { + "column": 45707, + "rms_error": 0.06260437138713795, + "maximum_absolute_error": 0.296875 + }, + { + "column": 309, + "rms_error": 0.06213507184944917, + "maximum_absolute_error": 0.3125 + }, + { + "column": 18169, + "rms_error": 0.06194062067037831, + "maximum_absolute_error": 0.3173828125 + }, + { + "column": 39382, + "rms_error": 0.06176609269320477, + "maximum_absolute_error": 0.23828125 + }, + { + "column": 30270, + "rms_error": 0.06164624748222817, + "maximum_absolute_error": 0.3203125 + }, + { + "column": 98225, + "rms_error": 0.061520043554539855, + "maximum_absolute_error": 0.25 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-4-code-513", + "phase": "prompt", + "seconds": 230.26318679191172, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "83013e7646760ead9206c4a53b7cac56a19d11035629c876f9ea6701dddedc08", + "rounded_reference_logits_sha256": "61e6cd3d56a41b198ee9877a9a477ec61310ea4826869eef057da42cd8549808", + "token_fixture_sha256": "c932ae46fd83f6e834a4001bd27496af3ebf54ac841a694d03a64558ba0daacb", + "original_ideal_fp64_differences": [ + { + "row": 264, + "native_top": 207, + "reference_top": 2993, + "native_maximum_set": [ + 207 + ], + "reference_maximum_set": [ + 2993 + ], + "maximum_sets_overlap": false, + "native_margin": 0.25, + "reference_margin": 0.375, + "native_reference_contender_gap": 0.5, + "reference_native_contender_gap": 0.375, + "rounded_reference_top": 207, + "rounded_reference_margin": 0.25, + "rounded_reference_native_contender_gap": 0.5625, + "rounded_reference_maximum_set": [ + 207 + ] + } + ] + }, + { + "rows": 257, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.008228574718582158, + "maximum_absolute_error": 0.40234375, + "differences": [], + "worst_rows": [ + { + "row": 9, + "rms_error": 0.11675100600170359, + "maximum_absolute_error": 0.40234375 + }, + { + "row": 72, + "rms_error": 0.09182025948456946, + "maximum_absolute_error": 0.34375 + }, + { + "row": 30, + "rms_error": 0.08301323679998208, + "maximum_absolute_error": 0.2890625 + }, + { + "row": 34, + "rms_error": 0.08161881181226428, + "maximum_absolute_error": 0.234375 + }, + { + "row": 142, + "rms_error": 0.07404224594098688, + "maximum_absolute_error": 0.1875 + }, + { + "row": 250, + "rms_error": 0.06245770977221806, + "maximum_absolute_error": 0.171875 + }, + { + "row": 196, + "rms_error": 0.05974744907730566, + "maximum_absolute_error": 0.15625 + }, + { + "row": 181, + "rms_error": 0.05968768378833964, + "maximum_absolute_error": 0.15625 + }, + { + "row": 21, + "rms_error": 0.059383517969852855, + "maximum_absolute_error": 0.21875 + }, + { + "row": 141, + "rms_error": 0.05867767050374448, + "maximum_absolute_error": 0.203125 + }, + { + "row": 189, + "rms_error": 0.05833769401635122, + "maximum_absolute_error": 0.1875 + }, + { + "row": 29, + "rms_error": 0.05833556144228005, + "maximum_absolute_error": 0.21875 + }, + { + "row": 115, + "rms_error": 0.057935115778285945, + "maximum_absolute_error": 0.1640625 + }, + { + "row": 129, + "rms_error": 0.057801159107845855, + "maximum_absolute_error": 0.171875 + }, + { + "row": 106, + "rms_error": 0.05778176406071792, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 107, + "rms_error": 0.05745385391010654, + "maximum_absolute_error": 0.25 + }, + { + "row": 63, + "rms_error": 0.05726298677642242, + "maximum_absolute_error": 0.25 + }, + { + "row": 85, + "rms_error": 0.05675869907432556, + "maximum_absolute_error": 0.203125 + }, + { + "row": 150, + "rms_error": 0.05647164936702668, + "maximum_absolute_error": 0.21875 + }, + { + "row": 125, + "rms_error": 0.056325135119553775, + "maximum_absolute_error": 0.15625 + } + ], + "worst_columns": [ + { + "column": 10625, + "rms_error": 0.0719168286234429, + "maximum_absolute_error": 0.3125 + }, + { + "column": 16420, + "rms_error": 0.06935220883801685, + "maximum_absolute_error": 0.3125 + }, + { + "column": 7161, + "rms_error": 0.06872535279635689, + "maximum_absolute_error": 0.28125 + }, + { + "column": 334, + "rms_error": 0.06852811266791806, + "maximum_absolute_error": 0.29296875 + }, + { + "column": 1840, + "rms_error": 0.06808063619059351, + "maximum_absolute_error": 0.34375 + }, + { + "column": 50474, + "rms_error": 0.06801655566196134, + "maximum_absolute_error": 0.34375 + }, + { + "column": 229, + "rms_error": 0.06790218938914094, + "maximum_absolute_error": 0.28125 + }, + { + "column": 10494, + "rms_error": 0.06760618030686806, + "maximum_absolute_error": 0.296875 + }, + { + "column": 3464, + "rms_error": 0.06581870418435679, + "maximum_absolute_error": 0.28125 + }, + { + "column": 278, + "rms_error": 0.06574005733457104, + "maximum_absolute_error": 0.359375 + }, + { + "column": 677, + "rms_error": 0.0644634874054111, + "maximum_absolute_error": 0.21875 + }, + { + "column": 2925, + "rms_error": 0.06352312981336071, + "maximum_absolute_error": 0.28125 + }, + { + "column": 90208, + "rms_error": 0.06342204452206861, + "maximum_absolute_error": 0.34375 + }, + { + "column": 3431, + "rms_error": 0.0632776334042306, + "maximum_absolute_error": 0.265625 + }, + { + "column": 3300, + "rms_error": 0.0631922709910923, + "maximum_absolute_error": 0.203125 + }, + { + "column": 3498, + "rms_error": 0.06288495654896703, + "maximum_absolute_error": 0.21875 + }, + { + "column": 32564, + "rms_error": 0.062457497914349076, + "maximum_absolute_error": 0.28125 + }, + { + "column": 405, + "rms_error": 0.06221173376451575, + "maximum_absolute_error": 0.296875 + }, + { + "column": 20, + "rms_error": 0.06212506474275177, + "maximum_absolute_error": 0.21875 + }, + { + "column": 91312, + "rms_error": 0.061823161313847616, + "maximum_absolute_error": 0.21875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-4-reasoning-257", + "phase": "prompt", + "seconds": 0.12575800018385053, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "118440ccd7fc07a502485a3266a2878ec815f316bcfa9dddd859e5d324fb8654", + "rounded_reference_logits_sha256": "b59c43083bfc6008233e7e249ced2f589ace92e1461e95c65300141aea14ab2b", + "token_fixture_sha256": "81449804fdd1a104d6dc9607c27d5cea49237064e50c80a765f68d8eba57f596", + "original_ideal_fp64_differences": [ + { + "row": 17, + "native_top": 296, + "reference_top": 405, + "native_maximum_set": [ + 296 + ], + "reference_maximum_set": [ + 405 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.125, + "native_reference_contender_gap": 0.1875, + "reference_native_contender_gap": 0.125, + "rounded_reference_top": 296, + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": 0.0625, + "rounded_reference_maximum_set": [ + 296 + ] + }, + { + "row": 18, + "native_top": 22, + "reference_top": 1018, + "native_maximum_set": [ + 22, + 1018 + ], + "reference_maximum_set": [ + 1018 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625, + "rounded_reference_top": 22, + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": 0.0625, + "rounded_reference_maximum_set": [ + 22 + ] + } + ] + }, + { + "rows": 513, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.00808784933410427, + "maximum_absolute_error": 0.4375, + "differences": [], + "worst_rows": [ + { + "row": 9, + "rms_error": 0.11675100600170359, + "maximum_absolute_error": 0.40234375 + }, + { + "row": 72, + "rms_error": 0.09182025948456946, + "maximum_absolute_error": 0.34375 + }, + { + "row": 303, + "rms_error": 0.08465159739842856, + "maximum_absolute_error": 0.203125 + }, + { + "row": 278, + "rms_error": 0.08344110807218878, + "maximum_absolute_error": 0.25 + }, + { + "row": 30, + "rms_error": 0.08301323679998208, + "maximum_absolute_error": 0.2890625 + }, + { + "row": 292, + "rms_error": 0.08177847174299556, + "maximum_absolute_error": 0.21875 + }, + { + "row": 34, + "rms_error": 0.08161881181226428, + "maximum_absolute_error": 0.234375 + }, + { + "row": 375, + "rms_error": 0.07934752642150551, + "maximum_absolute_error": 0.1875 + }, + { + "row": 492, + "rms_error": 0.07797467828138362, + "maximum_absolute_error": 0.1875 + }, + { + "row": 467, + "rms_error": 0.0765648229527427, + "maximum_absolute_error": 0.19921875 + }, + { + "row": 142, + "rms_error": 0.07404224594098688, + "maximum_absolute_error": 0.1875 + }, + { + "row": 274, + "rms_error": 0.06918524276020961, + "maximum_absolute_error": 0.171875 + }, + { + "row": 364, + "rms_error": 0.06769287746103053, + "maximum_absolute_error": 0.1875 + }, + { + "row": 448, + "rms_error": 0.067583798073357, + "maximum_absolute_error": 0.1875 + }, + { + "row": 496, + "rms_error": 0.0640855225590863, + "maximum_absolute_error": 0.1796875 + }, + { + "row": 270, + "rms_error": 0.06382185836130466, + "maximum_absolute_error": 0.15625 + }, + { + "row": 414, + "rms_error": 0.06344076185171192, + "maximum_absolute_error": 0.1875 + }, + { + "row": 387, + "rms_error": 0.06313538070881776, + "maximum_absolute_error": 0.171875 + }, + { + "row": 250, + "rms_error": 0.06245770977221806, + "maximum_absolute_error": 0.171875 + }, + { + "row": 264, + "rms_error": 0.062373291518676134, + "maximum_absolute_error": 0.16796875 + } + ], + "worst_columns": [ + { + "column": 10625, + "rms_error": 0.06729076776326634, + "maximum_absolute_error": 0.3125 + }, + { + "column": 7161, + "rms_error": 0.06409665907461286, + "maximum_absolute_error": 0.28125 + }, + { + "column": 229, + "rms_error": 0.06391387225951636, + "maximum_absolute_error": 0.28125 + }, + { + "column": 334, + "rms_error": 0.06390340853688993, + "maximum_absolute_error": 0.29296875 + }, + { + "column": 1840, + "rms_error": 0.06347278853165797, + "maximum_absolute_error": 0.34375 + }, + { + "column": 2925, + "rms_error": 0.062458180315551434, + "maximum_absolute_error": 0.4375 + }, + { + "column": 16420, + "rms_error": 0.06238160465035213, + "maximum_absolute_error": 0.3125 + }, + { + "column": 278, + "rms_error": 0.06203815034906912, + "maximum_absolute_error": 0.359375 + }, + { + "column": 677, + "rms_error": 0.06185531888774638, + "maximum_absolute_error": 0.21875 + }, + { + "column": 10494, + "rms_error": 0.06110432423620328, + "maximum_absolute_error": 0.296875 + }, + { + "column": 3431, + "rms_error": 0.06063245984212859, + "maximum_absolute_error": 0.265625 + }, + { + "column": 3498, + "rms_error": 0.05967599368262706, + "maximum_absolute_error": 0.25 + }, + { + "column": 50474, + "rms_error": 0.059501322355054294, + "maximum_absolute_error": 0.34375 + }, + { + "column": 3464, + "rms_error": 0.05877318220432772, + "maximum_absolute_error": 0.28125 + }, + { + "column": 8, + "rms_error": 0.05794794575442929, + "maximum_absolute_error": 0.21875 + }, + { + "column": 117761, + "rms_error": 0.05761925464253013, + "maximum_absolute_error": 0.25 + }, + { + "column": 90208, + "rms_error": 0.05761629296059998, + "maximum_absolute_error": 0.34375 + }, + { + "column": 3300, + "rms_error": 0.057612160837007945, + "maximum_absolute_error": 0.203125 + }, + { + "column": 43345, + "rms_error": 0.05732740572335475, + "maximum_absolute_error": 0.265625 + }, + { + "column": 20, + "rms_error": 0.05707476134976221, + "maximum_absolute_error": 0.21875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-4-reasoning-513", + "phase": "prompt", + "seconds": 231.08551929192618, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "7a2673d808c3fb08e430dccd995f02a0458d3b9a57342a985364f1542b73c13a", + "rounded_reference_logits_sha256": "5f59240c111ff17b863202b4dde8ee682e0591642ffd1bac989418fca6707e56", + "token_fixture_sha256": "c8ce58a8d512375ea4635bc85d0ea8a235506aa56026087bda216ea61357a863", + "original_ideal_fp64_differences": [ + { + "row": 17, + "native_top": 296, + "reference_top": 405, + "native_maximum_set": [ + 296 + ], + "reference_maximum_set": [ + 405 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.125, + "native_reference_contender_gap": 0.1875, + "reference_native_contender_gap": 0.125, + "rounded_reference_top": 296, + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": 0.0625, + "rounded_reference_maximum_set": [ + 296 + ] + }, + { + "row": 18, + "native_top": 22, + "reference_top": 1018, + "native_maximum_set": [ + 22, + 1018 + ], + "reference_maximum_set": [ + 1018 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625, + "rounded_reference_top": 22, + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": 0.0625, + "rounded_reference_maximum_set": [ + 22 + ] + } + ] + }, + { + "rows": 257, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "relative_rms": 0.014473017342804723, + "maximum_absolute_error": 0.875, + "differences": [ + { + "row": 13, + "native_top": 22, + "reference_top": 63908, + "native_maximum_set": [ + 22, + 63908 + ], + "reference_maximum_set": [ + 63908 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 22, + "native_top": 16033, + "reference_top": 91, + "native_maximum_set": [ + 16033 + ], + "reference_maximum_set": [ + 91, + 16033 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 83, + "rms_error": 0.1496681162289443, + "maximum_absolute_error": 0.875 + }, + { + "row": 120, + "rms_error": 0.13032353681397132, + "maximum_absolute_error": 0.28125 + }, + { + "row": 141, + "rms_error": 0.10617832077490623, + "maximum_absolute_error": 0.3046875 + }, + { + "row": 79, + "rms_error": 0.09618230871080037, + "maximum_absolute_error": 0.25 + }, + { + "row": 57, + "rms_error": 0.09000787683806573, + "maximum_absolute_error": 0.625 + }, + { + "row": 111, + "rms_error": 0.08984078273872659, + "maximum_absolute_error": 0.21875 + }, + { + "row": 10, + "rms_error": 0.08800637679348153, + "maximum_absolute_error": 0.234375 + }, + { + "row": 161, + "rms_error": 0.08595653681696512, + "maximum_absolute_error": 0.515625 + }, + { + "row": 164, + "rms_error": 0.08402424381687046, + "maximum_absolute_error": 0.2734375 + }, + { + "row": 169, + "rms_error": 0.0812521491756504, + "maximum_absolute_error": 0.234375 + }, + { + "row": 54, + "rms_error": 0.07760607841137634, + "maximum_absolute_error": 0.375 + }, + { + "row": 98, + "rms_error": 0.07496353208396284, + "maximum_absolute_error": 0.625 + }, + { + "row": 95, + "rms_error": 0.0730375653803272, + "maximum_absolute_error": 0.390625 + }, + { + "row": 128, + "rms_error": 0.07019072481617666, + "maximum_absolute_error": 0.19140625 + }, + { + "row": 246, + "rms_error": 0.06591859240343742, + "maximum_absolute_error": 0.203125 + }, + { + "row": 222, + "rms_error": 0.06346149897759733, + "maximum_absolute_error": 0.1640625 + }, + { + "row": 206, + "rms_error": 0.06305829751414915, + "maximum_absolute_error": 0.40625 + }, + { + "row": 110, + "rms_error": 0.06281539565759484, + "maximum_absolute_error": 0.21875 + }, + { + "row": 180, + "rms_error": 0.061961919282112435, + "maximum_absolute_error": 0.34375 + }, + { + "row": 183, + "rms_error": 0.061588703551617774, + "maximum_absolute_error": 0.166015625 + } + ], + "worst_columns": [ + { + "column": 33237, + "rms_error": 0.09271428661695896, + "maximum_absolute_error": 0.4375 + }, + { + "column": 283, + "rms_error": 0.08692508244560647, + "maximum_absolute_error": 0.875 + }, + { + "column": 16033, + "rms_error": 0.08644866858342837, + "maximum_absolute_error": 0.34375 + }, + { + "column": 295, + "rms_error": 0.0844347635346529, + "maximum_absolute_error": 0.484375 + }, + { + "column": 723, + "rms_error": 0.08408752939786455, + "maximum_absolute_error": 0.4375 + }, + { + "column": 6986, + "rms_error": 0.08205637459595938, + "maximum_absolute_error": 0.5 + }, + { + "column": 274, + "rms_error": 0.08145350774263904, + "maximum_absolute_error": 0.671875 + }, + { + "column": 116016, + "rms_error": 0.07966200971734298, + "maximum_absolute_error": 0.3125 + }, + { + "column": 291, + "rms_error": 0.07936417009370202, + "maximum_absolute_error": 0.7265625 + }, + { + "column": 376, + "rms_error": 0.07878119016004734, + "maximum_absolute_error": 0.55078125 + }, + { + "column": 61089, + "rms_error": 0.07843875483181038, + "maximum_absolute_error": 0.25 + }, + { + "column": 42847, + "rms_error": 0.07807788224830392, + "maximum_absolute_error": 0.421875 + }, + { + "column": 20975, + "rms_error": 0.07667168390156584, + "maximum_absolute_error": 0.515625 + }, + { + "column": 372, + "rms_error": 0.07619522316918846, + "maximum_absolute_error": 0.8359375 + }, + { + "column": 30270, + "rms_error": 0.07573778778260018, + "maximum_absolute_error": 0.34375 + }, + { + "column": 106633, + "rms_error": 0.07564334184440837, + "maximum_absolute_error": 0.3125 + }, + { + "column": 39382, + "rms_error": 0.07532347806087861, + "maximum_absolute_error": 0.328125 + }, + { + "column": 1224, + "rms_error": 0.07474792436959363, + "maximum_absolute_error": 0.4375 + }, + { + "column": 15960, + "rms_error": 0.0738545374795119, + "maximum_absolute_error": 0.392578125 + }, + { + "column": 410, + "rms_error": 0.07373929878906844, + "maximum_absolute_error": 0.78515625 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-code-257", + "phase": "prompt", + "seconds": 101.30898025026545, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "93ea83c90c5623f4e17502a526e2358f9e1ba3f2d03999295745257dc95bbca6", + "rounded_reference_logits_sha256": "de6e3637ed096da44629921c0360788270e32c5e516fcb6fd16515524b835d9f", + "token_fixture_sha256": "96859365056d513a414e6eeef9d4a85bf72c3f9630d3e13a9ea0f57dd81b8bea", + "original_ideal_fp64_differences": [ + { + "row": 13, + "native_top": 22, + "reference_top": 63908, + "native_maximum_set": [ + 22, + 63908 + ], + "reference_maximum_set": [ + 63908 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625, + "rounded_reference_top": 63908, + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": -0.0625, + "rounded_reference_maximum_set": [ + 63908 + ] + }, + { + "row": 22, + "native_top": 16033, + "reference_top": 91, + "native_maximum_set": [ + 16033 + ], + "reference_maximum_set": [ + 91 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.1875, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.1875, + "rounded_reference_top": 91, + "rounded_reference_margin": 0.0, + "rounded_reference_native_contender_gap": 0.0, + "rounded_reference_maximum_set": [ + 91, + 16033 + ] + }, + { + "row": 83, + "native_top": 3570, + "reference_top": 496, + "native_maximum_set": [ + 3570 + ], + "reference_maximum_set": [ + 496, + 3570 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.1875, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 3570, + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": 0.1875, + "rounded_reference_maximum_set": [ + 3570 + ] + } + ] + }, + { + "rows": 513, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "relative_rms": 0.013020523891176231, + "maximum_absolute_error": 0.875, + "differences": [ + { + "row": 13, + "native_top": 22, + "reference_top": 63908, + "native_maximum_set": [ + 22, + 63908 + ], + "reference_maximum_set": [ + 63908 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 22, + "native_top": 16033, + "reference_top": 91, + "native_maximum_set": [ + 16033 + ], + "reference_maximum_set": [ + 91, + 16033 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 83, + "rms_error": 0.1496681162289443, + "maximum_absolute_error": 0.875 + }, + { + "row": 120, + "rms_error": 0.13032353681397132, + "maximum_absolute_error": 0.28125 + }, + { + "row": 141, + "rms_error": 0.10617832077490623, + "maximum_absolute_error": 0.3046875 + }, + { + "row": 79, + "rms_error": 0.09618230871080037, + "maximum_absolute_error": 0.25 + }, + { + "row": 57, + "rms_error": 0.09000787683806573, + "maximum_absolute_error": 0.625 + }, + { + "row": 111, + "rms_error": 0.08984078273872659, + "maximum_absolute_error": 0.21875 + }, + { + "row": 10, + "rms_error": 0.08800637679348153, + "maximum_absolute_error": 0.234375 + }, + { + "row": 161, + "rms_error": 0.08595653681696512, + "maximum_absolute_error": 0.515625 + }, + { + "row": 164, + "rms_error": 0.08402424381687046, + "maximum_absolute_error": 0.2734375 + }, + { + "row": 329, + "rms_error": 0.0834715978102895, + "maximum_absolute_error": 0.25 + }, + { + "row": 284, + "rms_error": 0.0830134199145985, + "maximum_absolute_error": 0.5625 + }, + { + "row": 169, + "rms_error": 0.0812521491756504, + "maximum_absolute_error": 0.234375 + }, + { + "row": 54, + "rms_error": 0.07760607841137634, + "maximum_absolute_error": 0.375 + }, + { + "row": 306, + "rms_error": 0.07714252956909921, + "maximum_absolute_error": 0.1875 + }, + { + "row": 98, + "rms_error": 0.07496353208396284, + "maximum_absolute_error": 0.625 + }, + { + "row": 95, + "rms_error": 0.0730375653803272, + "maximum_absolute_error": 0.390625 + }, + { + "row": 128, + "rms_error": 0.07019072481617666, + "maximum_absolute_error": 0.19140625 + }, + { + "row": 246, + "rms_error": 0.06591859240343742, + "maximum_absolute_error": 0.203125 + }, + { + "row": 426, + "rms_error": 0.06451268485095099, + "maximum_absolute_error": 0.53125 + }, + { + "row": 222, + "rms_error": 0.06346149897759733, + "maximum_absolute_error": 0.1640625 + } + ], + "worst_columns": [ + { + "column": 20975, + "rms_error": 0.08632377977370913, + "maximum_absolute_error": 0.5625 + }, + { + "column": 33237, + "rms_error": 0.08500419421181206, + "maximum_absolute_error": 0.4375 + }, + { + "column": 16033, + "rms_error": 0.0765510358679167, + "maximum_absolute_error": 0.34375 + }, + { + "column": 295, + "rms_error": 0.07522592140902692, + "maximum_absolute_error": 0.484375 + }, + { + "column": 22497, + "rms_error": 0.07519884631857517, + "maximum_absolute_error": 0.546875 + }, + { + "column": 5272, + "rms_error": 0.07465740928606568, + "maximum_absolute_error": 0.390625 + }, + { + "column": 723, + "rms_error": 0.07428088570898066, + "maximum_absolute_error": 0.4375 + }, + { + "column": 283, + "rms_error": 0.07284057711943236, + "maximum_absolute_error": 0.875 + }, + { + "column": 15960, + "rms_error": 0.0722710020677921, + "maximum_absolute_error": 0.392578125 + }, + { + "column": 61089, + "rms_error": 0.0709506068651178, + "maximum_absolute_error": 0.25 + }, + { + "column": 116016, + "rms_error": 0.0708318892633313, + "maximum_absolute_error": 0.3125 + }, + { + "column": 274, + "rms_error": 0.0695850200345508, + "maximum_absolute_error": 0.671875 + }, + { + "column": 495, + "rms_error": 0.0691785208124479, + "maximum_absolute_error": 0.375 + }, + { + "column": 8, + "rms_error": 0.06915724626986011, + "maximum_absolute_error": 0.25 + }, + { + "column": 6986, + "rms_error": 0.06898611913786616, + "maximum_absolute_error": 0.5 + }, + { + "column": 30270, + "rms_error": 0.06892099221502541, + "maximum_absolute_error": 0.34375 + }, + { + "column": 39382, + "rms_error": 0.06783539013587425, + "maximum_absolute_error": 0.328125 + }, + { + "column": 376, + "rms_error": 0.06752820128542786, + "maximum_absolute_error": 0.55078125 + }, + { + "column": 93017, + "rms_error": 0.06720480366411694, + "maximum_absolute_error": 0.25 + }, + { + "column": 106633, + "rms_error": 0.06715214487485084, + "maximum_absolute_error": 0.3125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-code-513", + "phase": "prompt", + "seconds": 194.07661470817402, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "3acb11b05fae7d193a867401fe358871782c9d63d2955eb77686cf09f0e97799", + "rounded_reference_logits_sha256": "429f1bf013d0b795e0a10773fb4d3f43e20d6115506936724d555f049dfa5384", + "token_fixture_sha256": "c932ae46fd83f6e834a4001bd27496af3ebf54ac841a694d03a64558ba0daacb", + "original_ideal_fp64_differences": [ + { + "row": 13, + "native_top": 22, + "reference_top": 63908, + "native_maximum_set": [ + 22, + 63908 + ], + "reference_maximum_set": [ + 63908 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625, + "rounded_reference_top": 63908, + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": -0.0625, + "rounded_reference_maximum_set": [ + 63908 + ] + }, + { + "row": 22, + "native_top": 16033, + "reference_top": 91, + "native_maximum_set": [ + 16033 + ], + "reference_maximum_set": [ + 91 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.1875, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.1875, + "rounded_reference_top": 91, + "rounded_reference_margin": 0.0, + "rounded_reference_native_contender_gap": 0.0, + "rounded_reference_maximum_set": [ + 91, + 16033 + ] + }, + { + "row": 83, + "native_top": 3570, + "reference_top": 496, + "native_maximum_set": [ + 3570 + ], + "reference_maximum_set": [ + 496, + 3570 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.1875, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 3570, + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": 0.1875, + "rounded_reference_maximum_set": [ + 3570 + ] + } + ] + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "relative_rms": 0.013509190417066086, + "maximum_absolute_error": 0.625, + "differences": [ + { + "row": 13, + "native_top": 22, + "reference_top": 63908, + "native_maximum_set": [ + 22, + 63908 + ], + "reference_maximum_set": [ + 63908 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 22, + "native_top": 16033, + "reference_top": 91, + "native_maximum_set": [ + 16033 + ], + "reference_maximum_set": [ + 91, + 16033 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 57, + "rms_error": 0.09000787683806573, + "maximum_absolute_error": 0.625 + }, + { + "row": 10, + "rms_error": 0.08800637679348153, + "maximum_absolute_error": 0.234375 + }, + { + "row": 54, + "rms_error": 0.07760607841137634, + "maximum_absolute_error": 0.375 + }, + { + "row": 36, + "rms_error": 0.058355442854574543, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 53, + "rms_error": 0.057159352869539715, + "maximum_absolute_error": 0.21875 + }, + { + "row": 20, + "rms_error": 0.055633565618538675, + "maximum_absolute_error": 0.296875 + }, + { + "row": 39, + "rms_error": 0.053347736683817384, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 13, + "rms_error": 0.052123672609206625, + "maximum_absolute_error": 0.21875 + }, + { + "row": 8, + "rms_error": 0.05116690334326474, + "maximum_absolute_error": 0.25 + }, + { + "row": 2, + "rms_error": 0.05008304981339593, + "maximum_absolute_error": 0.125 + }, + { + "row": 52, + "rms_error": 0.04974962183100654, + "maximum_absolute_error": 0.1875 + }, + { + "row": 62, + "rms_error": 0.04902776384179979, + "maximum_absolute_error": 0.21875 + }, + { + "row": 55, + "rms_error": 0.047242131140363866, + "maximum_absolute_error": 0.21875 + }, + { + "row": 23, + "rms_error": 0.044142037283058606, + "maximum_absolute_error": 0.1875 + }, + { + "row": 11, + "rms_error": 0.043190213412404824, + "maximum_absolute_error": 0.25 + }, + { + "row": 22, + "rms_error": 0.04095067360254933, + "maximum_absolute_error": 0.3125 + }, + { + "row": 38, + "rms_error": 0.04044440498995601, + "maximum_absolute_error": 0.234375 + }, + { + "row": 30, + "rms_error": 0.04023781052786102, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 50, + "rms_error": 0.0400162730201504, + "maximum_absolute_error": 0.171875 + }, + { + "row": 26, + "rms_error": 0.03986160470382173, + "maximum_absolute_error": 0.25 + } + ], + "worst_columns": [ + { + "column": 538, + "rms_error": 0.09985948465502727, + "maximum_absolute_error": 0.625 + }, + { + "column": 33237, + "rms_error": 0.09714630618708237, + "maximum_absolute_error": 0.375 + }, + { + "column": 723, + "rms_error": 0.09163830636319745, + "maximum_absolute_error": 0.4375 + }, + { + "column": 16033, + "rms_error": 0.0907320804033186, + "maximum_absolute_error": 0.34375 + }, + { + "column": 6986, + "rms_error": 0.0893078048926329, + "maximum_absolute_error": 0.28125 + }, + { + "column": 30270, + "rms_error": 0.08854197407562756, + "maximum_absolute_error": 0.34375 + }, + { + "column": 9700, + "rms_error": 0.0861797386586336, + "maximum_absolute_error": 0.3125 + }, + { + "column": 61089, + "rms_error": 0.08516518525177089, + "maximum_absolute_error": 0.25 + }, + { + "column": 116016, + "rms_error": 0.08419055912772351, + "maximum_absolute_error": 0.3125 + }, + { + "column": 29956, + "rms_error": 0.08386143846355991, + "maximum_absolute_error": 0.28125 + }, + { + "column": 39382, + "rms_error": 0.0833193654285085, + "maximum_absolute_error": 0.328125 + }, + { + "column": 274, + "rms_error": 0.08328050464397146, + "maximum_absolute_error": 0.46875 + }, + { + "column": 6907, + "rms_error": 0.08258207761026035, + "maximum_absolute_error": 0.552734375 + }, + { + "column": 42847, + "rms_error": 0.08252694023323655, + "maximum_absolute_error": 0.296875 + }, + { + "column": 1583, + "rms_error": 0.08156546550782054, + "maximum_absolute_error": 0.25 + }, + { + "column": 295, + "rms_error": 0.08148593392714178, + "maximum_absolute_error": 0.4375 + }, + { + "column": 717, + "rms_error": 0.08125915112683267, + "maximum_absolute_error": 0.515625 + }, + { + "column": 93538, + "rms_error": 0.08125813356366732, + "maximum_absolute_error": 0.46875 + }, + { + "column": 2741, + "rms_error": 0.08083424732039579, + "maximum_absolute_error": 0.45703125 + }, + { + "column": 346, + "rms_error": 0.08050453816084938, + "maximum_absolute_error": 0.5 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-code-64", + "phase": "prompt", + "seconds": 28.386479707900435, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "02225f1c8c7f5de05ebd44bd49948ad000aac040f3d6025d6655a9772ed08f9b", + "rounded_reference_logits_sha256": "3f061bbfb2d31d235f0c4c5681e170f1bb2fe5f928ff805bf768d7727787e614", + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "original_ideal_fp64_differences": [ + { + "row": 13, + "native_top": 22, + "reference_top": 63908, + "native_maximum_set": [ + 22, + 63908 + ], + "reference_maximum_set": [ + 63908 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625, + "rounded_reference_top": 63908, + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": -0.0625, + "rounded_reference_maximum_set": [ + 63908 + ] + }, + { + "row": 22, + "native_top": 16033, + "reference_top": 91, + "native_maximum_set": [ + 16033 + ], + "reference_maximum_set": [ + 91 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.1875, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.1875, + "rounded_reference_top": 91, + "rounded_reference_margin": 0.0, + "rounded_reference_native_contender_gap": 0.0, + "rounded_reference_maximum_set": [ + 91, + 16033 + ] + } + ] + }, + { + "rows": 257, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "relative_rms": 0.009405403303912105, + "maximum_absolute_error": 0.328125, + "differences": [ + { + "row": 32, + "native_top": 3420, + "reference_top": 1030, + "native_maximum_set": [ + 3420 + ], + "reference_maximum_set": [ + 1030, + 3420 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + }, + { + "row": 38, + "native_top": 22, + "reference_top": 20, + "native_maximum_set": [ + 22 + ], + "reference_maximum_set": [ + 20, + 22 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 25, + "rms_error": 0.0975424054281447, + "maximum_absolute_error": 0.27734375 + }, + { + "row": 58, + "rms_error": 0.09720541341469165, + "maximum_absolute_error": 0.2890625 + }, + { + "row": 167, + "rms_error": 0.08541813750329376, + "maximum_absolute_error": 0.203125 + }, + { + "row": 71, + "rms_error": 0.08222472966558243, + "maximum_absolute_error": 0.21875 + }, + { + "row": 69, + "rms_error": 0.07900262953197698, + "maximum_absolute_error": 0.21875 + }, + { + "row": 23, + "rms_error": 0.07823154942174776, + "maximum_absolute_error": 0.28125 + }, + { + "row": 46, + "rms_error": 0.07619000753995392, + "maximum_absolute_error": 0.3125 + }, + { + "row": 205, + "rms_error": 0.07389922043343698, + "maximum_absolute_error": 0.15625 + }, + { + "row": 120, + "rms_error": 0.07318591150742734, + "maximum_absolute_error": 0.203125 + }, + { + "row": 138, + "rms_error": 0.07282438448104953, + "maximum_absolute_error": 0.1875 + }, + { + "row": 15, + "rms_error": 0.06945189203516938, + "maximum_absolute_error": 0.234375 + }, + { + "row": 162, + "rms_error": 0.06739418268668715, + "maximum_absolute_error": 0.15625 + }, + { + "row": 117, + "rms_error": 0.0673446920131743, + "maximum_absolute_error": 0.1875 + }, + { + "row": 95, + "rms_error": 0.06529522346039617, + "maximum_absolute_error": 0.1728515625 + }, + { + "row": 243, + "rms_error": 0.06323140369357666, + "maximum_absolute_error": 0.15625 + }, + { + "row": 136, + "rms_error": 0.062167744150499464, + "maximum_absolute_error": 0.1640625 + }, + { + "row": 199, + "rms_error": 0.06044723492513211, + "maximum_absolute_error": 0.328125 + }, + { + "row": 34, + "rms_error": 0.05941716027318458, + "maximum_absolute_error": 0.234375 + }, + { + "row": 49, + "rms_error": 0.05803273067013118, + "maximum_absolute_error": 0.21875 + }, + { + "row": 93, + "rms_error": 0.0569628991739252, + "maximum_absolute_error": 0.19140625 + } + ], + "worst_columns": [ + { + "column": 3599, + "rms_error": 0.06628999817737627, + "maximum_absolute_error": 0.28125 + }, + { + "column": 13816, + "rms_error": 0.0657797969869993, + "maximum_absolute_error": 0.25 + }, + { + "column": 90978, + "rms_error": 0.06351774889629735, + "maximum_absolute_error": 0.234375 + }, + { + "column": 6986, + "rms_error": 0.062285190040124776, + "maximum_absolute_error": 0.25 + }, + { + "column": 42847, + "rms_error": 0.06124954557587252, + "maximum_absolute_error": 0.25 + }, + { + "column": 309, + "rms_error": 0.06056440082266772, + "maximum_absolute_error": 0.1875 + }, + { + "column": 6542, + "rms_error": 0.0604083516630064, + "maximum_absolute_error": 0.234375 + }, + { + "column": 30317, + "rms_error": 0.05994803423100661, + "maximum_absolute_error": 0.23828125 + }, + { + "column": 11448, + "rms_error": 0.05961544665209866, + "maximum_absolute_error": 0.21875 + }, + { + "column": 417, + "rms_error": 0.05900797106311973, + "maximum_absolute_error": 0.1796875 + }, + { + "column": 335, + "rms_error": 0.058938953757341685, + "maximum_absolute_error": 0.29296875 + }, + { + "column": 49422, + "rms_error": 0.05880309629929141, + "maximum_absolute_error": 0.328125 + }, + { + "column": 52072, + "rms_error": 0.0582079870594192, + "maximum_absolute_error": 0.25 + }, + { + "column": 3420, + "rms_error": 0.057983657926954944, + "maximum_absolute_error": 0.1875 + }, + { + "column": 9700, + "rms_error": 0.05766543832421079, + "maximum_absolute_error": 0.234375 + }, + { + "column": 11460, + "rms_error": 0.05761640860515311, + "maximum_absolute_error": 0.21875 + }, + { + "column": 267, + "rms_error": 0.05744674702545165, + "maximum_absolute_error": 0.1875 + }, + { + "column": 1030, + "rms_error": 0.057184439945468554, + "maximum_absolute_error": 0.28125 + }, + { + "column": 9011, + "rms_error": 0.05637369843227854, + "maximum_absolute_error": 0.1875 + }, + { + "column": 91384, + "rms_error": 0.05630617581521297, + "maximum_absolute_error": 0.265625 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-prose-257", + "phase": "prompt", + "seconds": 99.08814133284613, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "9847c355bb21bff7af4f9a9db7953de32e794976757ac442379e0a9e25b97e1a", + "rounded_reference_logits_sha256": "7504cf65f7511eab393f108f05ea7c83ba91c8f0a374f2d5f6888e9104437b6d", + "token_fixture_sha256": "522ec0f7a7ef8481d2f5533e3962ddad1f418d4337b095282e40c096e2befb1d", + "original_ideal_fp64_differences": [ + { + "row": 17, + "native_top": 3547, + "reference_top": 3983, + "native_maximum_set": [ + 3547 + ], + "reference_maximum_set": [ + 3983 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0625, + "rounded_reference_top": 3547, + "rounded_reference_margin": 0.125, + "rounded_reference_native_contender_gap": 0.125, + "rounded_reference_maximum_set": [ + 3547 + ] + }, + { + "row": 22, + "native_top": 22, + "reference_top": 20, + "native_maximum_set": [ + 22 + ], + "reference_maximum_set": [ + 20, + 22 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 22, + "rounded_reference_margin": 0.125, + "rounded_reference_native_contender_gap": 0.125, + "rounded_reference_maximum_set": [ + 22 + ] + }, + { + "row": 32, + "native_top": 3420, + "reference_top": 1030, + "native_maximum_set": [ + 3420 + ], + "reference_maximum_set": [ + 1030, + 3420 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 1030, + "rounded_reference_margin": 0.0, + "rounded_reference_native_contender_gap": 0.0, + "rounded_reference_maximum_set": [ + 1030, + 3420 + ] + }, + { + "row": 38, + "native_top": 22, + "reference_top": 20, + "native_maximum_set": [ + 22 + ], + "reference_maximum_set": [ + 20, + 22 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 20, + "rounded_reference_margin": 0.0, + "rounded_reference_native_contender_gap": 0.0, + "rounded_reference_maximum_set": [ + 20, + 22 + ] + } + ] + }, + { + "rows": 481, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "relative_rms": 0.008331668324461805, + "maximum_absolute_error": 0.328125, + "differences": [ + { + "row": 32, + "native_top": 3420, + "reference_top": 1030, + "native_maximum_set": [ + 3420 + ], + "reference_maximum_set": [ + 1030, + 3420 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + }, + { + "row": 38, + "native_top": 22, + "reference_top": 20, + "native_maximum_set": [ + 22 + ], + "reference_maximum_set": [ + 20, + 22 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 25, + "rms_error": 0.0975424054281447, + "maximum_absolute_error": 0.27734375 + }, + { + "row": 58, + "rms_error": 0.09720541341469165, + "maximum_absolute_error": 0.2890625 + }, + { + "row": 167, + "rms_error": 0.08541813750329376, + "maximum_absolute_error": 0.203125 + }, + { + "row": 71, + "rms_error": 0.08222472966558243, + "maximum_absolute_error": 0.21875 + }, + { + "row": 69, + "rms_error": 0.07900262953197698, + "maximum_absolute_error": 0.21875 + }, + { + "row": 23, + "rms_error": 0.07823154942174776, + "maximum_absolute_error": 0.28125 + }, + { + "row": 46, + "rms_error": 0.07619000753995392, + "maximum_absolute_error": 0.3125 + }, + { + "row": 205, + "rms_error": 0.07389922043343698, + "maximum_absolute_error": 0.15625 + }, + { + "row": 120, + "rms_error": 0.07318591150742734, + "maximum_absolute_error": 0.203125 + }, + { + "row": 138, + "rms_error": 0.07282438448104953, + "maximum_absolute_error": 0.1875 + }, + { + "row": 15, + "rms_error": 0.06945189203516938, + "maximum_absolute_error": 0.234375 + }, + { + "row": 162, + "rms_error": 0.06739418268668715, + "maximum_absolute_error": 0.15625 + }, + { + "row": 117, + "rms_error": 0.0673446920131743, + "maximum_absolute_error": 0.1875 + }, + { + "row": 95, + "rms_error": 0.06529522346039617, + "maximum_absolute_error": 0.1728515625 + }, + { + "row": 376, + "rms_error": 0.06398267242077037, + "maximum_absolute_error": 0.1650390625 + }, + { + "row": 243, + "rms_error": 0.06323140369357666, + "maximum_absolute_error": 0.15625 + }, + { + "row": 136, + "rms_error": 0.062167744150499464, + "maximum_absolute_error": 0.1640625 + }, + { + "row": 199, + "rms_error": 0.06044723492513211, + "maximum_absolute_error": 0.328125 + }, + { + "row": 34, + "rms_error": 0.05941716027318458, + "maximum_absolute_error": 0.234375 + }, + { + "row": 49, + "rms_error": 0.05803273067013118, + "maximum_absolute_error": 0.21875 + } + ], + "worst_columns": [ + { + "column": 13816, + "rms_error": 0.0592589493874173, + "maximum_absolute_error": 0.25 + }, + { + "column": 6986, + "rms_error": 0.059233276967287925, + "maximum_absolute_error": 0.25 + }, + { + "column": 417, + "rms_error": 0.058981110328195176, + "maximum_absolute_error": 0.265625 + }, + { + "column": 42847, + "rms_error": 0.058172815239817655, + "maximum_absolute_error": 0.25 + }, + { + "column": 3599, + "rms_error": 0.05710337107643613, + "maximum_absolute_error": 0.28125 + }, + { + "column": 90978, + "rms_error": 0.05703476027945669, + "maximum_absolute_error": 0.234375 + }, + { + "column": 49422, + "rms_error": 0.05694291164895438, + "maximum_absolute_error": 0.328125 + }, + { + "column": 309, + "rms_error": 0.05595820954144564, + "maximum_absolute_error": 0.1875 + }, + { + "column": 9700, + "rms_error": 0.05484103366022586, + "maximum_absolute_error": 0.234375 + }, + { + "column": 30317, + "rms_error": 0.054750470493591476, + "maximum_absolute_error": 0.23828125 + }, + { + "column": 6542, + "rms_error": 0.05453027284696916, + "maximum_absolute_error": 0.234375 + }, + { + "column": 335, + "rms_error": 0.054419571560981775, + "maximum_absolute_error": 0.29296875 + }, + { + "column": 11448, + "rms_error": 0.05438239188387566, + "maximum_absolute_error": 0.21875 + }, + { + "column": 278, + "rms_error": 0.05437904970282908, + "maximum_absolute_error": 0.1875 + }, + { + "column": 11460, + "rms_error": 0.054297909344293106, + "maximum_absolute_error": 0.21875 + }, + { + "column": 3420, + "rms_error": 0.05397241607939977, + "maximum_absolute_error": 0.1875 + }, + { + "column": 22, + "rms_error": 0.05386573519302058, + "maximum_absolute_error": 0.125 + }, + { + "column": 20, + "rms_error": 0.05310991738378729, + "maximum_absolute_error": 0.1875 + }, + { + "column": 9011, + "rms_error": 0.0530349147621503, + "maximum_absolute_error": 0.1875 + }, + { + "column": 29956, + "rms_error": 0.052558718578596404, + "maximum_absolute_error": 0.1875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-prose-513", + "phase": "prompt", + "seconds": 182.36508262483403, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "4633f10a1ef2861c9e8a6b6e2af3b001516d8f7dfacda7c74afb24ce0771a007", + "rounded_reference_logits_sha256": "20eb8081bc4fd23041c51d587314e0333de75fac33e910b4bd438c4b24cb7ae8", + "token_fixture_sha256": "4b9e533cc1948ee72c4231178ec3af21808d7542c97f35997bb9538161d9a4cb", + "original_ideal_fp64_differences": [ + { + "row": 17, + "native_top": 3547, + "reference_top": 3983, + "native_maximum_set": [ + 3547 + ], + "reference_maximum_set": [ + 3983 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0625, + "rounded_reference_top": 3547, + "rounded_reference_margin": 0.125, + "rounded_reference_native_contender_gap": 0.125, + "rounded_reference_maximum_set": [ + 3547 + ] + }, + { + "row": 22, + "native_top": 22, + "reference_top": 20, + "native_maximum_set": [ + 22 + ], + "reference_maximum_set": [ + 20, + 22 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 22, + "rounded_reference_margin": 0.125, + "rounded_reference_native_contender_gap": 0.125, + "rounded_reference_maximum_set": [ + 22 + ] + }, + { + "row": 32, + "native_top": 3420, + "reference_top": 1030, + "native_maximum_set": [ + 3420 + ], + "reference_maximum_set": [ + 1030, + 3420 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 1030, + "rounded_reference_margin": 0.0, + "rounded_reference_native_contender_gap": 0.0, + "rounded_reference_maximum_set": [ + 1030, + 3420 + ] + }, + { + "row": 38, + "native_top": 22, + "reference_top": 20, + "native_maximum_set": [ + 22 + ], + "reference_maximum_set": [ + 20, + 22 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 20, + "rounded_reference_margin": 0.0, + "rounded_reference_native_contender_gap": 0.0, + "rounded_reference_maximum_set": [ + 20, + 22 + ] + } + ] + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "relative_rms": 0.010846921926639188, + "maximum_absolute_error": 0.3125, + "differences": [ + { + "row": 32, + "native_top": 3420, + "reference_top": 1030, + "native_maximum_set": [ + 3420 + ], + "reference_maximum_set": [ + 1030, + 3420 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + }, + { + "row": 38, + "native_top": 22, + "reference_top": 20, + "native_maximum_set": [ + 22 + ], + "reference_maximum_set": [ + 20, + 22 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 25, + "rms_error": 0.0975424054281447, + "maximum_absolute_error": 0.27734375 + }, + { + "row": 58, + "rms_error": 0.09720541341469165, + "maximum_absolute_error": 0.2890625 + }, + { + "row": 23, + "rms_error": 0.07823154942174776, + "maximum_absolute_error": 0.28125 + }, + { + "row": 46, + "rms_error": 0.07619000753995392, + "maximum_absolute_error": 0.3125 + }, + { + "row": 15, + "rms_error": 0.06945189203516938, + "maximum_absolute_error": 0.234375 + }, + { + "row": 34, + "rms_error": 0.05941716027318458, + "maximum_absolute_error": 0.234375 + }, + { + "row": 49, + "rms_error": 0.05803273067013118, + "maximum_absolute_error": 0.21875 + }, + { + "row": 45, + "rms_error": 0.0525123345531119, + "maximum_absolute_error": 0.203125 + }, + { + "row": 61, + "rms_error": 0.052208496950912245, + "maximum_absolute_error": 0.1875 + }, + { + "row": 2, + "rms_error": 0.04931431873399461, + "maximum_absolute_error": 0.265625 + }, + { + "row": 16, + "rms_error": 0.0473177122392379, + "maximum_absolute_error": 0.21875 + }, + { + "row": 43, + "rms_error": 0.04563653066516502, + "maximum_absolute_error": 0.20849609375 + }, + { + "row": 41, + "rms_error": 0.044165791761466844, + "maximum_absolute_error": 0.21875 + }, + { + "row": 32, + "rms_error": 0.044100236673400975, + "maximum_absolute_error": 0.21875 + }, + { + "row": 51, + "rms_error": 0.043397627618378384, + "maximum_absolute_error": 0.15625 + }, + { + "row": 56, + "rms_error": 0.04327666267227569, + "maximum_absolute_error": 0.25 + }, + { + "row": 31, + "rms_error": 0.042430706374013645, + "maximum_absolute_error": 0.25 + }, + { + "row": 36, + "rms_error": 0.04189792981343197, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 18, + "rms_error": 0.04143660547812578, + "maximum_absolute_error": 0.21875 + }, + { + "row": 39, + "rms_error": 0.0407629949349385, + "maximum_absolute_error": 0.29296875 + } + ], + "worst_columns": [ + { + "column": 3599, + "rms_error": 0.0997470577351016, + "maximum_absolute_error": 0.28125 + }, + { + "column": 42847, + "rms_error": 0.08593298076537768, + "maximum_absolute_error": 0.25 + }, + { + "column": 90978, + "rms_error": 0.0833161978749641, + "maximum_absolute_error": 0.234375 + }, + { + "column": 6986, + "rms_error": 0.08306194111339649, + "maximum_absolute_error": 0.25 + }, + { + "column": 6542, + "rms_error": 0.0825293721307139, + "maximum_absolute_error": 0.234375 + }, + { + "column": 13816, + "rms_error": 0.08245315682711211, + "maximum_absolute_error": 0.25 + }, + { + "column": 8818, + "rms_error": 0.08131434318343182, + "maximum_absolute_error": 0.25 + }, + { + "column": 11448, + "rms_error": 0.08051850166276953, + "maximum_absolute_error": 0.21875 + }, + { + "column": 91384, + "rms_error": 0.0798754336579659, + "maximum_absolute_error": 0.265625 + }, + { + "column": 11353, + "rms_error": 0.0794464176388837, + "maximum_absolute_error": 0.25 + }, + { + "column": 9864, + "rms_error": 0.07928485591358113, + "maximum_absolute_error": 0.2421875 + }, + { + "column": 7986, + "rms_error": 0.07883987327456152, + "maximum_absolute_error": 0.25 + }, + { + "column": 52072, + "rms_error": 0.07816847545209255, + "maximum_absolute_error": 0.25 + }, + { + "column": 30317, + "rms_error": 0.07669826945083377, + "maximum_absolute_error": 0.23828125 + }, + { + "column": 417, + "rms_error": 0.07551562875216965, + "maximum_absolute_error": 0.1796875 + }, + { + "column": 66722, + "rms_error": 0.07508622537942632, + "maximum_absolute_error": 0.1875 + }, + { + "column": 974, + "rms_error": 0.0746126076861771, + "maximum_absolute_error": 0.1875 + }, + { + "column": 3507, + "rms_error": 0.07447335216638261, + "maximum_absolute_error": 0.2275390625 + }, + { + "column": 10776, + "rms_error": 0.07422437771993734, + "maximum_absolute_error": 0.1875 + }, + { + "column": 11783, + "rms_error": 0.07385389858429019, + "maximum_absolute_error": 0.2119140625 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-prose-64", + "phase": "prompt", + "seconds": 28.35513670835644, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "e494f59f8738cb12a4329b1ba0c034861541e2d11fc8b31401b9ed212f3c67ae", + "rounded_reference_logits_sha256": "accd330d760b5cd7c14e60c17f663fb5052637cbb8365c2cc81c47916f2abb89", + "token_fixture_sha256": "9c3636ee22fb5d1a9d4d6e3cc0b8e47cce20cb4288bb16f2970de8d55ee27708", + "original_ideal_fp64_differences": [ + { + "row": 17, + "native_top": 3547, + "reference_top": 3983, + "native_maximum_set": [ + 3547 + ], + "reference_maximum_set": [ + 3983 + ], + "maximum_sets_overlap": false, + "native_margin": 0.0625, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0625, + "rounded_reference_top": 3547, + "rounded_reference_margin": 0.125, + "rounded_reference_native_contender_gap": 0.125, + "rounded_reference_maximum_set": [ + 3547 + ] + }, + { + "row": 22, + "native_top": 22, + "reference_top": 20, + "native_maximum_set": [ + 22 + ], + "reference_maximum_set": [ + 20, + 22 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 22, + "rounded_reference_margin": 0.125, + "rounded_reference_native_contender_gap": 0.125, + "rounded_reference_maximum_set": [ + 22 + ] + }, + { + "row": 32, + "native_top": 3420, + "reference_top": 1030, + "native_maximum_set": [ + 3420 + ], + "reference_maximum_set": [ + 1030, + 3420 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 1030, + "rounded_reference_margin": 0.0, + "rounded_reference_native_contender_gap": 0.0, + "rounded_reference_maximum_set": [ + 1030, + 3420 + ] + }, + { + "row": 38, + "native_top": 22, + "reference_top": 20, + "native_maximum_set": [ + 22 + ], + "reference_maximum_set": [ + 20, + 22 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 20, + "rounded_reference_margin": 0.0, + "rounded_reference_native_contender_gap": 0.0, + "rounded_reference_maximum_set": [ + 20, + 22 + ] + } + ] + }, + { + "rows": 257, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 1, + "relative_rms": 0.008871270762820693, + "maximum_absolute_error": 0.59375, + "differences": [ + { + "row": 5, + "native_top": 334, + "reference_top": 440, + "native_maximum_set": [ + 334, + 440 + ], + "reference_maximum_set": [ + 440 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 24, + "native_top": 27, + "reference_top": 1325, + "native_maximum_set": [ + 27 + ], + "reference_maximum_set": [ + 1325 + ], + "maximum_sets_overlap": false, + "native_margin": 0.09375, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.09375, + "reference_native_contender_gap": 0.0625 + } + ], + "worst_rows": [ + { + "row": 48, + "rms_error": 0.11943096815557515, + "maximum_absolute_error": 0.25 + }, + { + "row": 110, + "rms_error": 0.09681007797730948, + "maximum_absolute_error": 0.59375 + }, + { + "row": 133, + "rms_error": 0.08738835840824201, + "maximum_absolute_error": 0.1875 + }, + { + "row": 225, + "rms_error": 0.08301502331520284, + "maximum_absolute_error": 0.171875 + }, + { + "row": 126, + "rms_error": 0.0818256122419536, + "maximum_absolute_error": 0.1875 + }, + { + "row": 178, + "rms_error": 0.08180875233687941, + "maximum_absolute_error": 0.3125 + }, + { + "row": 87, + "rms_error": 0.07598679763392681, + "maximum_absolute_error": 0.1875 + }, + { + "row": 127, + "rms_error": 0.07367216400391804, + "maximum_absolute_error": 0.20947265625 + }, + { + "row": 119, + "rms_error": 0.06849202956808123, + "maximum_absolute_error": 0.1875 + }, + { + "row": 145, + "rms_error": 0.06781027528078155, + "maximum_absolute_error": 0.1875 + }, + { + "row": 94, + "rms_error": 0.06703265838132924, + "maximum_absolute_error": 0.2109375 + }, + { + "row": 103, + "rms_error": 0.06622000854942998, + "maximum_absolute_error": 0.171875 + }, + { + "row": 42, + "rms_error": 0.06545042330967576, + "maximum_absolute_error": 0.1875 + }, + { + "row": 226, + "rms_error": 0.06540093962659949, + "maximum_absolute_error": 0.25 + }, + { + "row": 10, + "rms_error": 0.06415382028777855, + "maximum_absolute_error": 0.197265625 + }, + { + "row": 192, + "rms_error": 0.0620348815408526, + "maximum_absolute_error": 0.1875 + }, + { + "row": 83, + "rms_error": 0.061385422318241185, + "maximum_absolute_error": 0.2109375 + }, + { + "row": 181, + "rms_error": 0.060845000676773446, + "maximum_absolute_error": 0.1484375 + }, + { + "row": 122, + "rms_error": 0.05832380868246905, + "maximum_absolute_error": 0.19140625 + }, + { + "row": 19, + "rms_error": 0.058249381173734315, + "maximum_absolute_error": 0.21875 + } + ], + "worst_columns": [ + { + "column": 334, + "rms_error": 0.07127854868662369, + "maximum_absolute_error": 0.28125 + }, + { + "column": 1703, + "rms_error": 0.06915155998653433, + "maximum_absolute_error": 0.28125 + }, + { + "column": 1008, + "rms_error": 0.06870391348118576, + "maximum_absolute_error": 0.40625 + }, + { + "column": 677, + "rms_error": 0.06833561001252067, + "maximum_absolute_error": 0.25 + }, + { + "column": 229, + "rms_error": 0.06792491961926284, + "maximum_absolute_error": 0.25 + }, + { + "column": 1196, + "rms_error": 0.0668850933553704, + "maximum_absolute_error": 0.25 + }, + { + "column": 10625, + "rms_error": 0.06680857192944044, + "maximum_absolute_error": 0.28125 + }, + { + "column": 16420, + "rms_error": 0.0660919916244234, + "maximum_absolute_error": 0.4375 + }, + { + "column": 10494, + "rms_error": 0.06558272912483111, + "maximum_absolute_error": 0.25 + }, + { + "column": 1840, + "rms_error": 0.06350464963029116, + "maximum_absolute_error": 0.25 + }, + { + "column": 3498, + "rms_error": 0.06311282209255421, + "maximum_absolute_error": 0.375 + }, + { + "column": 278, + "rms_error": 0.0628344718381305, + "maximum_absolute_error": 0.375 + }, + { + "column": 22, + "rms_error": 0.06246638600122314, + "maximum_absolute_error": 0.1875 + }, + { + "column": 9798, + "rms_error": 0.062411143541799635, + "maximum_absolute_error": 0.25 + }, + { + "column": 117761, + "rms_error": 0.062247165541258996, + "maximum_absolute_error": 0.22265625 + }, + { + "column": 7161, + "rms_error": 0.06184455280197585, + "maximum_absolute_error": 0.234375 + }, + { + "column": 405, + "rms_error": 0.061591887268323195, + "maximum_absolute_error": 0.25 + }, + { + "column": 3464, + "rms_error": 0.061570103723194075, + "maximum_absolute_error": 0.21875 + }, + { + "column": 3431, + "rms_error": 0.06129315398921641, + "maximum_absolute_error": 0.25 + }, + { + "column": 4475, + "rms_error": 0.06117482727100836, + "maximum_absolute_error": 0.25 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-reasoning-257", + "phase": "prompt", + "seconds": 100.29989004088566, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "9be9b95af0ff0c00697f8737ce9271ccac0e6dcd519b56d49f3666623ae6fd2c", + "rounded_reference_logits_sha256": "17c0f9bf2e32a742f3fe710aefde3c74f62ffc9516bd789fd502690a0e5f8ac8", + "token_fixture_sha256": "81449804fdd1a104d6dc9607c27d5cea49237064e50c80a765f68d8eba57f596", + "original_ideal_fp64_differences": [ + { + "row": 5, + "native_top": 334, + "reference_top": 440, + "native_maximum_set": [ + 334, + 440 + ], + "reference_maximum_set": [ + 440 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625, + "rounded_reference_top": 440, + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": -0.0625, + "rounded_reference_maximum_set": [ + 440 + ] + }, + { + "row": 24, + "native_top": 27, + "reference_top": 1325, + "native_maximum_set": [ + 27 + ], + "reference_maximum_set": [ + 1325 + ], + "maximum_sets_overlap": false, + "native_margin": 0.09375, + "reference_margin": 0.15625, + "native_reference_contender_gap": 0.09375, + "reference_native_contender_gap": 0.15625, + "rounded_reference_top": 1325, + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": -0.0625, + "rounded_reference_maximum_set": [ + 1325 + ] + } + ] + }, + { + "rows": 513, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 1, + "relative_rms": 0.00853242478815418, + "maximum_absolute_error": 0.59375, + "differences": [ + { + "row": 5, + "native_top": 334, + "reference_top": 440, + "native_maximum_set": [ + 334, + 440 + ], + "reference_maximum_set": [ + 440 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 24, + "native_top": 27, + "reference_top": 1325, + "native_maximum_set": [ + 27 + ], + "reference_maximum_set": [ + 1325 + ], + "maximum_sets_overlap": false, + "native_margin": 0.09375, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.09375, + "reference_native_contender_gap": 0.0625 + } + ], + "worst_rows": [ + { + "row": 48, + "rms_error": 0.11943096815557515, + "maximum_absolute_error": 0.25 + }, + { + "row": 266, + "rms_error": 0.10762707057459482, + "maximum_absolute_error": 0.375 + }, + { + "row": 110, + "rms_error": 0.09681007797730948, + "maximum_absolute_error": 0.59375 + }, + { + "row": 336, + "rms_error": 0.09443384299139349, + "maximum_absolute_error": 0.28125 + }, + { + "row": 375, + "rms_error": 0.0943641610560971, + "maximum_absolute_error": 0.25 + }, + { + "row": 395, + "rms_error": 0.0937203364550311, + "maximum_absolute_error": 0.203125 + }, + { + "row": 133, + "rms_error": 0.08738835840824201, + "maximum_absolute_error": 0.1875 + }, + { + "row": 225, + "rms_error": 0.08301502331520284, + "maximum_absolute_error": 0.171875 + }, + { + "row": 126, + "rms_error": 0.0818256122419536, + "maximum_absolute_error": 0.1875 + }, + { + "row": 178, + "rms_error": 0.08180875233687941, + "maximum_absolute_error": 0.3125 + }, + { + "row": 87, + "rms_error": 0.07598679763392681, + "maximum_absolute_error": 0.1875 + }, + { + "row": 127, + "rms_error": 0.07367216400391804, + "maximum_absolute_error": 0.20947265625 + }, + { + "row": 409, + "rms_error": 0.07276205024770893, + "maximum_absolute_error": 0.171875 + }, + { + "row": 119, + "rms_error": 0.06849202956808123, + "maximum_absolute_error": 0.1875 + }, + { + "row": 145, + "rms_error": 0.06781027528078155, + "maximum_absolute_error": 0.1875 + }, + { + "row": 408, + "rms_error": 0.06712484598592339, + "maximum_absolute_error": 0.19921875 + }, + { + "row": 94, + "rms_error": 0.06703265838132924, + "maximum_absolute_error": 0.2109375 + }, + { + "row": 426, + "rms_error": 0.0663119644262462, + "maximum_absolute_error": 0.19140625 + }, + { + "row": 103, + "rms_error": 0.06622000854942998, + "maximum_absolute_error": 0.171875 + }, + { + "row": 42, + "rms_error": 0.06545042330967576, + "maximum_absolute_error": 0.1875 + } + ], + "worst_columns": [ + { + "column": 677, + "rms_error": 0.06787327900614677, + "maximum_absolute_error": 0.28125 + }, + { + "column": 1196, + "rms_error": 0.06698574087552416, + "maximum_absolute_error": 0.34375 + }, + { + "column": 334, + "rms_error": 0.06472283387443853, + "maximum_absolute_error": 0.28125 + }, + { + "column": 1008, + "rms_error": 0.0642480412572785, + "maximum_absolute_error": 0.40625 + }, + { + "column": 229, + "rms_error": 0.06341020825721838, + "maximum_absolute_error": 0.25 + }, + { + "column": 10625, + "rms_error": 0.06329557764159505, + "maximum_absolute_error": 0.28125 + }, + { + "column": 278, + "rms_error": 0.06312084620261127, + "maximum_absolute_error": 0.375 + }, + { + "column": 16420, + "rms_error": 0.061956931632575624, + "maximum_absolute_error": 0.4375 + }, + { + "column": 551, + "rms_error": 0.061952914297441465, + "maximum_absolute_error": 0.25 + }, + { + "column": 1703, + "rms_error": 0.061305721116854416, + "maximum_absolute_error": 0.28125 + }, + { + "column": 1840, + "rms_error": 0.060937261111432384, + "maximum_absolute_error": 0.25 + }, + { + "column": 3498, + "rms_error": 0.06052103884807295, + "maximum_absolute_error": 0.375 + }, + { + "column": 10494, + "rms_error": 0.05988106544329477, + "maximum_absolute_error": 0.25 + }, + { + "column": 405, + "rms_error": 0.05949597771653368, + "maximum_absolute_error": 0.3125 + }, + { + "column": 1394, + "rms_error": 0.05936334663329307, + "maximum_absolute_error": 0.25 + }, + { + "column": 22, + "rms_error": 0.0592344986158477, + "maximum_absolute_error": 0.1875 + }, + { + "column": 8, + "rms_error": 0.05843920762461839, + "maximum_absolute_error": 0.2109375 + }, + { + "column": 207, + "rms_error": 0.0570259156732694, + "maximum_absolute_error": 0.2109375 + }, + { + "column": 93017, + "rms_error": 0.05701596513810726, + "maximum_absolute_error": 0.1875 + }, + { + "column": 7161, + "rms_error": 0.05691991118497564, + "maximum_absolute_error": 0.25 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-reasoning-513", + "phase": "prompt", + "seconds": 194.67192812496796, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "049ebf779f8b00545a73253fb30fcd55cb1343321cb2f035fd5ad725153ddb66", + "rounded_reference_logits_sha256": "8a33db8fed54139070a816278126d3b841e2ad6f45ec1949b755751cf35619b9", + "token_fixture_sha256": "c8ce58a8d512375ea4635bc85d0ea8a235506aa56026087bda216ea61357a863", + "original_ideal_fp64_differences": [ + { + "row": 5, + "native_top": 334, + "reference_top": 440, + "native_maximum_set": [ + 334, + 440 + ], + "reference_maximum_set": [ + 440 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625, + "rounded_reference_top": 440, + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": -0.0625, + "rounded_reference_maximum_set": [ + 440 + ] + }, + { + "row": 24, + "native_top": 27, + "reference_top": 1325, + "native_maximum_set": [ + 27 + ], + "reference_maximum_set": [ + 1325 + ], + "maximum_sets_overlap": false, + "native_margin": 0.09375, + "reference_margin": 0.15625, + "native_reference_contender_gap": 0.09375, + "reference_native_contender_gap": 0.15625, + "rounded_reference_top": 1325, + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": -0.0625, + "rounded_reference_maximum_set": [ + 1325 + ] + } + ] + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 1, + "relative_rms": 0.007795208001208681, + "maximum_absolute_error": 0.3125, + "differences": [ + { + "row": 5, + "native_top": 334, + "reference_top": 440, + "native_maximum_set": [ + 334, + 440 + ], + "reference_maximum_set": [ + 440 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 24, + "native_top": 27, + "reference_top": 1325, + "native_maximum_set": [ + 27 + ], + "reference_maximum_set": [ + 1325 + ], + "maximum_sets_overlap": false, + "native_margin": 0.09375, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.09375, + "reference_native_contender_gap": 0.0625 + } + ], + "worst_rows": [ + { + "row": 48, + "rms_error": 0.11943096815557515, + "maximum_absolute_error": 0.25 + }, + { + "row": 42, + "rms_error": 0.06545042330967576, + "maximum_absolute_error": 0.1875 + }, + { + "row": 10, + "rms_error": 0.06415382028777855, + "maximum_absolute_error": 0.197265625 + }, + { + "row": 19, + "rms_error": 0.058249381173734315, + "maximum_absolute_error": 0.21875 + }, + { + "row": 21, + "rms_error": 0.05328268357571703, + "maximum_absolute_error": 0.203125 + }, + { + "row": 9, + "rms_error": 0.05258708141270272, + "maximum_absolute_error": 0.125 + }, + { + "row": 62, + "rms_error": 0.05062880335757404, + "maximum_absolute_error": 0.1875 + }, + { + "row": 23, + "rms_error": 0.04903957403813111, + "maximum_absolute_error": 0.15625 + }, + { + "row": 57, + "rms_error": 0.04866191696151736, + "maximum_absolute_error": 0.1875 + }, + { + "row": 43, + "rms_error": 0.04845294730993942, + "maximum_absolute_error": 0.1875 + }, + { + "row": 50, + "rms_error": 0.04749265067039846, + "maximum_absolute_error": 0.15625 + }, + { + "row": 63, + "rms_error": 0.047448822798662416, + "maximum_absolute_error": 0.1875 + }, + { + "row": 31, + "rms_error": 0.04706801249044121, + "maximum_absolute_error": 0.203125 + }, + { + "row": 39, + "rms_error": 0.04415255307457535, + "maximum_absolute_error": 0.21875 + }, + { + "row": 41, + "rms_error": 0.044101410208917814, + "maximum_absolute_error": 0.15625 + }, + { + "row": 24, + "rms_error": 0.04054052759953391, + "maximum_absolute_error": 0.21875 + }, + { + "row": 61, + "rms_error": 0.03808155268935307, + "maximum_absolute_error": 0.1875 + }, + { + "row": 32, + "rms_error": 0.037493528678949284, + "maximum_absolute_error": 0.234375 + }, + { + "row": 37, + "rms_error": 0.03658707169943915, + "maximum_absolute_error": 0.15625 + }, + { + "row": 6, + "rms_error": 0.03657214432162103, + "maximum_absolute_error": 0.1328125 + } + ], + "worst_columns": [ + { + "column": 229, + "rms_error": 0.07691475933295944, + "maximum_absolute_error": 0.25 + }, + { + "column": 1703, + "rms_error": 0.07072497288476526, + "maximum_absolute_error": 0.1875 + }, + { + "column": 1196, + "rms_error": 0.07004751500167694, + "maximum_absolute_error": 0.1875 + }, + { + "column": 10494, + "rms_error": 0.06984982321190494, + "maximum_absolute_error": 0.1875 + }, + { + "column": 20, + "rms_error": 0.06934281435967474, + "maximum_absolute_error": 0.1875 + }, + { + "column": 2258, + "rms_error": 0.06893979663775722, + "maximum_absolute_error": 0.3125 + }, + { + "column": 10941, + "rms_error": 0.06810015638956954, + "maximum_absolute_error": 0.3125 + }, + { + "column": 7161, + "rms_error": 0.06787120352537211, + "maximum_absolute_error": 0.21875 + }, + { + "column": 677, + "rms_error": 0.06709915385274173, + "maximum_absolute_error": 0.15625 + }, + { + "column": 334, + "rms_error": 0.06583892179990534, + "maximum_absolute_error": 0.21875 + }, + { + "column": 9931, + "rms_error": 0.06480874273159347, + "maximum_absolute_error": 0.1875 + }, + { + "column": 105706, + "rms_error": 0.06295041861057851, + "maximum_absolute_error": 0.1875 + }, + { + "column": 22, + "rms_error": 0.062033341610330764, + "maximum_absolute_error": 0.15625 + }, + { + "column": 11966, + "rms_error": 0.062024212877659815, + "maximum_absolute_error": 0.21875 + }, + { + "column": 10625, + "rms_error": 0.0616969734114209, + "maximum_absolute_error": 0.20703125 + }, + { + "column": 105616, + "rms_error": 0.0615098726365345, + "maximum_absolute_error": 0.15625 + }, + { + "column": 207, + "rms_error": 0.06122163894480857, + "maximum_absolute_error": 0.15625 + }, + { + "column": 98325, + "rms_error": 0.061082362158973974, + "maximum_absolute_error": 0.1875 + }, + { + "column": 1840, + "rms_error": 0.060932041804081266, + "maximum_absolute_error": 0.21875 + }, + { + "column": 16449, + "rms_error": 0.060918423723606806, + "maximum_absolute_error": 0.15625 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-reasoning-64", + "phase": "prompt", + "seconds": 28.617688458878547, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "128b568000f7ac49b194d340bfea43df9950db1dbadc3da8983b2100252da128", + "rounded_reference_logits_sha256": "09fbd18e90de0d457895481ecfe3c258e531215932f28785aa1bb535e904c16a", + "token_fixture_sha256": "f3b50ade94353156b86075d186c03eae5a91e007593c314c55e19575b9c8f8eb", + "original_ideal_fp64_differences": [ + { + "row": 5, + "native_top": 334, + "reference_top": 440, + "native_maximum_set": [ + 334, + 440 + ], + "reference_maximum_set": [ + 440 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625, + "rounded_reference_top": 440, + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": -0.0625, + "rounded_reference_maximum_set": [ + 440 + ] + }, + { + "row": 24, + "native_top": 27, + "reference_top": 1325, + "native_maximum_set": [ + 27 + ], + "reference_maximum_set": [ + 1325 + ], + "maximum_sets_overlap": false, + "native_margin": 0.09375, + "reference_margin": 0.15625, + "native_reference_contender_gap": 0.09375, + "reference_native_contender_gap": 0.15625, + "rounded_reference_top": 1325, + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": -0.0625, + "rounded_reference_maximum_set": [ + 1325 + ] + } + ] + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.010676928309154964, + "maximum_absolute_error": 0.384765625, + "differences": [], + "worst_rows": [ + { + "row": 33, + "rms_error": 0.11715401059319087, + "maximum_absolute_error": 0.375 + }, + { + "row": 62, + "rms_error": 0.07064930423914317, + "maximum_absolute_error": 0.2265625 + }, + { + "row": 8, + "rms_error": 0.0660147669051244, + "maximum_absolute_error": 0.21875 + }, + { + "row": 25, + "rms_error": 0.06550749873901603, + "maximum_absolute_error": 0.203125 + }, + { + "row": 57, + "rms_error": 0.06495587979105812, + "maximum_absolute_error": 0.25 + }, + { + "row": 39, + "rms_error": 0.06408834047397304, + "maximum_absolute_error": 0.2421875 + }, + { + "row": 56, + "rms_error": 0.06314352660754154, + "maximum_absolute_error": 0.21484375 + }, + { + "row": 22, + "rms_error": 0.061153545980802, + "maximum_absolute_error": 0.28125 + }, + { + "row": 21, + "rms_error": 0.06102433500212631, + "maximum_absolute_error": 0.384765625 + }, + { + "row": 13, + "rms_error": 0.05812695813926894, + "maximum_absolute_error": 0.18243408203125 + }, + { + "row": 61, + "rms_error": 0.05701862398353163, + "maximum_absolute_error": 0.1640625 + }, + { + "row": 27, + "rms_error": 0.05475311854235568, + "maximum_absolute_error": 0.15625 + }, + { + "row": 46, + "rms_error": 0.05260661617521811, + "maximum_absolute_error": 0.228515625 + }, + { + "row": 45, + "rms_error": 0.052194217819958776, + "maximum_absolute_error": 0.208984375 + }, + { + "row": 53, + "rms_error": 0.05089795385947298, + "maximum_absolute_error": 0.1875 + }, + { + "row": 52, + "rms_error": 0.044354264472818275, + "maximum_absolute_error": 0.15625 + }, + { + "row": 20, + "rms_error": 0.04418983921681325, + "maximum_absolute_error": 0.247314453125 + }, + { + "row": 15, + "rms_error": 0.041442567292812725, + "maximum_absolute_error": 0.25 + }, + { + "row": 18, + "rms_error": 0.04098034058836111, + "maximum_absolute_error": 0.1796875 + }, + { + "row": 63, + "rms_error": 0.04049172481998616, + "maximum_absolute_error": 0.1875 + } + ], + "worst_columns": [ + { + "column": 3599, + "rms_error": 0.08793779273109813, + "maximum_absolute_error": 0.2734375 + }, + { + "column": 6542, + "rms_error": 0.0821685550161909, + "maximum_absolute_error": 0.375 + }, + { + "column": 7986, + "rms_error": 0.08095505958486879, + "maximum_absolute_error": 0.28125 + }, + { + "column": 49422, + "rms_error": 0.08060444558443788, + "maximum_absolute_error": 0.25 + }, + { + "column": 97278, + "rms_error": 0.07578679286027144, + "maximum_absolute_error": 0.2265625 + }, + { + "column": 3420, + "rms_error": 0.07228379407793313, + "maximum_absolute_error": 0.25 + }, + { + "column": 8818, + "rms_error": 0.07188990583518166, + "maximum_absolute_error": 0.296875 + }, + { + "column": 9263, + "rms_error": 0.07167550082544015, + "maximum_absolute_error": 0.35205078125 + }, + { + "column": 47374, + "rms_error": 0.07151350199236312, + "maximum_absolute_error": 0.21875 + }, + { + "column": 10265, + "rms_error": 0.07129569966323307, + "maximum_absolute_error": 0.384765625 + }, + { + "column": 422, + "rms_error": 0.06987645289545404, + "maximum_absolute_error": 0.1796875 + }, + { + "column": 4718, + "rms_error": 0.06979144267442448, + "maximum_absolute_error": 0.21875 + }, + { + "column": 9864, + "rms_error": 0.06935135953609109, + "maximum_absolute_error": 0.203125 + }, + { + "column": 1030, + "rms_error": 0.06930670323395263, + "maximum_absolute_error": 0.1875 + }, + { + "column": 267, + "rms_error": 0.06889439072053621, + "maximum_absolute_error": 0.34375 + }, + { + "column": 24475, + "rms_error": 0.06887243379634232, + "maximum_absolute_error": 0.34375 + }, + { + "column": 117325, + "rms_error": 0.06865820456854378, + "maximum_absolute_error": 0.25 + }, + { + "column": 95484, + "rms_error": 0.0685070078529889, + "maximum_absolute_error": 0.1875 + }, + { + "column": 34263, + "rms_error": 0.06844254611193758, + "maximum_absolute_error": 0.34375 + }, + { + "column": 18352, + "rms_error": 0.06842275346661701, + "maximum_absolute_error": 0.3642578125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-8-prose-64", + "phase": "prompt", + "seconds": 29.39553208416328, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "e4374b6b58d2f8863dbac5560eb80c9e3d8f47c767b8c98d4617884d7700f686", + "rounded_reference_logits_sha256": "efc889882c8c14d329fc1fcd799e49fc45c837e060f767333833b0dd1f7c4eef", + "token_fixture_sha256": "9c3636ee22fb5d1a9d4d6e3cc0b8e47cce20cb4288bb16f2970de8d55ee27708", + "original_ideal_fp64_differences": [ + { + "row": 0, + "native_top": 278, + "reference_top": 20, + "native_maximum_set": [ + 278 + ], + "reference_maximum_set": [ + 20, + 278 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 278, + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": 0.0625, + "rounded_reference_maximum_set": [ + 278 + ] + }, + { + "row": 10, + "native_top": 10513, + "reference_top": 4986, + "native_maximum_set": [ + 10513 + ], + "reference_maximum_set": [ + 4986, + 10513 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 10513, + "rounded_reference_margin": 0.125, + "rounded_reference_native_contender_gap": 0.125, + "rounded_reference_maximum_set": [ + 10513 + ] + } + ] + }, + { + "rows": 513, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.008981459929561184, + "maximum_absolute_error": 0.625, + "differences": [], + "worst_rows": [ + { + "row": 453, + "rms_error": 0.12919133095517948, + "maximum_absolute_error": 0.28125 + }, + { + "row": 336, + "rms_error": 0.11239795215540833, + "maximum_absolute_error": 0.28125 + }, + { + "row": 219, + "rms_error": 0.10839130383615313, + "maximum_absolute_error": 0.25 + }, + { + "row": 225, + "rms_error": 0.10427399855522272, + "maximum_absolute_error": 0.203125 + }, + { + "row": 96, + "rms_error": 0.10308223039787269, + "maximum_absolute_error": 0.3125 + }, + { + "row": 216, + "rms_error": 0.10161562775639274, + "maximum_absolute_error": 0.2734375 + }, + { + "row": 106, + "rms_error": 0.10114591250604772, + "maximum_absolute_error": 0.390625 + }, + { + "row": 21, + "rms_error": 0.09373265363494243, + "maximum_absolute_error": 0.2265625 + }, + { + "row": 43, + "rms_error": 0.09186032800438218, + "maximum_absolute_error": 0.3046875 + }, + { + "row": 415, + "rms_error": 0.08432257342607874, + "maximum_absolute_error": 0.171875 + }, + { + "row": 48, + "rms_error": 0.08219678506739612, + "maximum_absolute_error": 0.375 + }, + { + "row": 126, + "rms_error": 0.0818408991425119, + "maximum_absolute_error": 0.22265625 + }, + { + "row": 69, + "rms_error": 0.07966679594709014, + "maximum_absolute_error": 0.25 + }, + { + "row": 255, + "rms_error": 0.0738938933132548, + "maximum_absolute_error": 0.625 + }, + { + "row": 484, + "rms_error": 0.07374336991093611, + "maximum_absolute_error": 0.1875 + }, + { + "row": 343, + "rms_error": 0.07362876158705046, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 265, + "rms_error": 0.07307294779612346, + "maximum_absolute_error": 0.21875 + }, + { + "row": 414, + "rms_error": 0.07244367012788763, + "maximum_absolute_error": 0.1875 + }, + { + "row": 36, + "rms_error": 0.0704002108511174, + "maximum_absolute_error": 0.21875 + }, + { + "row": 51, + "rms_error": 0.07024464365859565, + "maximum_absolute_error": 0.21875 + } + ], + "worst_columns": [ + { + "column": 334, + "rms_error": 0.08306270849169323, + "maximum_absolute_error": 0.5 + }, + { + "column": 1840, + "rms_error": 0.07130993129891984, + "maximum_absolute_error": 0.34375 + }, + { + "column": 1703, + "rms_error": 0.07095154254005155, + "maximum_absolute_error": 0.375 + }, + { + "column": 267, + "rms_error": 0.06854453784852053, + "maximum_absolute_error": 0.4375 + }, + { + "column": 16420, + "rms_error": 0.06850919075528122, + "maximum_absolute_error": 0.46875 + }, + { + "column": 10625, + "rms_error": 0.06666656865006765, + "maximum_absolute_error": 0.25 + }, + { + "column": 9798, + "rms_error": 0.06623526108428945, + "maximum_absolute_error": 0.46875 + }, + { + "column": 229, + "rms_error": 0.0659755284451623, + "maximum_absolute_error": 0.28125 + }, + { + "column": 405, + "rms_error": 0.0658930299967849, + "maximum_absolute_error": 0.25 + }, + { + "column": 3498, + "rms_error": 0.06570068797305167, + "maximum_absolute_error": 0.375 + }, + { + "column": 20, + "rms_error": 0.0646180683214382, + "maximum_absolute_error": 0.25 + }, + { + "column": 4475, + "rms_error": 0.06458650694102266, + "maximum_absolute_error": 0.625 + }, + { + "column": 278, + "rms_error": 0.06441487519160408, + "maximum_absolute_error": 0.375 + }, + { + "column": 677, + "rms_error": 0.0640602359254666, + "maximum_absolute_error": 0.21875 + }, + { + "column": 10494, + "rms_error": 0.06352734121374307, + "maximum_absolute_error": 0.1953125 + }, + { + "column": 22, + "rms_error": 0.06252338945893869, + "maximum_absolute_error": 0.1875 + }, + { + "column": 6026, + "rms_error": 0.06236869746688868, + "maximum_absolute_error": 0.40625 + }, + { + "column": 1196, + "rms_error": 0.062111120003132285, + "maximum_absolute_error": 0.3125 + }, + { + "column": 2925, + "rms_error": 0.062081794966746685, + "maximum_absolute_error": 0.375 + }, + { + "column": 5435, + "rms_error": 0.06190424021952325, + "maximum_absolute_error": 0.375 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-8-reasoning-513", + "phase": "prompt", + "seconds": 193.77772895805538, + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "0fbe7f9ff161bf9b0961bd4d64b081f775dc51bb8eb72e7c1bcef7ac9bcbd582", + "rounded_reference_logits_sha256": "499abbaa05178bc8aa6375bd17848fb1e472f8267a6cc62e74bb826b7f47afe6", + "token_fixture_sha256": "c8ce58a8d512375ea4635bc85d0ea8a235506aa56026087bda216ea61357a863", + "original_ideal_fp64_differences": [ + { + "row": 5, + "native_top": 440, + "reference_top": 334, + "native_maximum_set": [ + 440 + ], + "reference_maximum_set": [ + 334, + 440 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0, + "rounded_reference_top": 440, + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": 0.0625, + "rounded_reference_maximum_set": [ + 440 + ] + }, + { + "row": 33, + "native_top": 1579, + "reference_top": 2925, + "native_maximum_set": [ + 1579, + 2925 + ], + "reference_maximum_set": [ + 2925 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625, + "rounded_reference_top": 1579, + "rounded_reference_margin": 0.0, + "rounded_reference_native_contender_gap": 0.0, + "rounded_reference_maximum_set": [ + 1579, + 2925 + ] + } + ] + } + ], + "rows": 5581, + "top1_differences": 22, + "disjoint_maximum_sets": 3 +} diff --git a/docs/receipts/lfm3/rounded-fp64-resolution.json b/docs/receipts/lfm3/rounded-fp64-resolution.json new file mode 100644 index 0000000000..365963fb89 --- /dev/null +++ b/docs/receipts/lfm3/rounded-fp64-resolution.json @@ -0,0 +1,847 @@ +{ + "scope": "Recorded ideal-FP64 differences are retained separately; independent reference applies only named production rounding sites, with no tolerance or automatic acceptance", + "affine_only": { + "cells": 17, + "rows": 5581, + "top1_differences": 22, + "disjoint_maximum_sets": 3 + }, + "guided_norm_conv_cases": [ + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.008043711960330414, + "maximum_absolute_error": 0.34375, + "differences": [ + { + "row": 6, + "native_top": 9798, + "reference_top": 9743, + "native_maximum_set": [ + 9798 + ], + "reference_maximum_set": [ + 9743, + 9798 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 51, + "rms_error": 0.11861646575515228, + "maximum_absolute_error": 0.34375 + }, + { + "row": 10, + "rms_error": 0.10028614776804698, + "maximum_absolute_error": 0.21875 + }, + { + "row": 57, + "rms_error": 0.0861529837832831, + "maximum_absolute_error": 0.21875 + }, + { + "row": 24, + "rms_error": 0.05687998224278724, + "maximum_absolute_error": 0.1875 + }, + { + "row": 48, + "rms_error": 0.05600190458937229, + "maximum_absolute_error": 0.1875 + }, + { + "row": 60, + "rms_error": 0.05498064187219183, + "maximum_absolute_error": 0.3125 + }, + { + "row": 32, + "rms_error": 0.05294731569054374, + "maximum_absolute_error": 0.21875 + }, + { + "row": 17, + "rms_error": 0.04935522540415026, + "maximum_absolute_error": 0.15625 + }, + { + "row": 50, + "rms_error": 0.0478751943152622, + "maximum_absolute_error": 0.171875 + }, + { + "row": 30, + "rms_error": 0.047753962581513175, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 9, + "rms_error": 0.04555611920115846, + "maximum_absolute_error": 0.1875 + }, + { + "row": 31, + "rms_error": 0.045409813623385815, + "maximum_absolute_error": 0.1640625 + }, + { + "row": 56, + "rms_error": 0.04311155120797251, + "maximum_absolute_error": 0.203125 + }, + { + "row": 62, + "rms_error": 0.04123161695385987, + "maximum_absolute_error": 0.171875 + }, + { + "row": 45, + "rms_error": 0.04050624266366054, + "maximum_absolute_error": 0.1875 + }, + { + "row": 42, + "rms_error": 0.038526539946546666, + "maximum_absolute_error": 0.125 + }, + { + "row": 53, + "rms_error": 0.037666491313599185, + "maximum_absolute_error": 0.15625 + }, + { + "row": 63, + "rms_error": 0.03744615993758534, + "maximum_absolute_error": 0.125 + }, + { + "row": 44, + "rms_error": 0.037078881417759536, + "maximum_absolute_error": 0.203125 + }, + { + "row": 28, + "rms_error": 0.03621204994136106, + "maximum_absolute_error": 0.15625 + } + ], + "worst_columns": [ + { + "column": 7161, + "rms_error": 0.0739487202983055, + "maximum_absolute_error": 0.203125 + }, + { + "column": 229, + "rms_error": 0.07305314144244157, + "maximum_absolute_error": 0.1875 + }, + { + "column": 1840, + "rms_error": 0.07213394953111382, + "maximum_absolute_error": 0.3125 + }, + { + "column": 10494, + "rms_error": 0.07134923616253058, + "maximum_absolute_error": 0.28125 + }, + { + "column": 10625, + "rms_error": 0.07134766978366326, + "maximum_absolute_error": 0.2578125 + }, + { + "column": 3431, + "rms_error": 0.07027679916665594, + "maximum_absolute_error": 0.1875 + }, + { + "column": 15114, + "rms_error": 0.06613980864073528, + "maximum_absolute_error": 0.33984375 + }, + { + "column": 117113, + "rms_error": 0.06395546022850564, + "maximum_absolute_error": 0.1875 + }, + { + "column": 5037, + "rms_error": 0.0637027232793923, + "maximum_absolute_error": 0.296875 + }, + { + "column": 1703, + "rms_error": 0.06359664123111303, + "maximum_absolute_error": 0.1875 + }, + { + "column": 4827, + "rms_error": 0.06355588798115991, + "maximum_absolute_error": 0.234375 + }, + { + "column": 8187, + "rms_error": 0.06341566551278807, + "maximum_absolute_error": 0.3203125 + }, + { + "column": 42985, + "rms_error": 0.06277423398584353, + "maximum_absolute_error": 0.2109375 + }, + { + "column": 36483, + "rms_error": 0.06250661250613916, + "maximum_absolute_error": 0.15625 + }, + { + "column": 62333, + "rms_error": 0.06225591465536169, + "maximum_absolute_error": 0.1875 + }, + { + "column": 398, + "rms_error": 0.0621247416424273, + "maximum_absolute_error": 0.2734375 + }, + { + "column": 677, + "rms_error": 0.061515686515717274, + "maximum_absolute_error": 0.1875 + }, + { + "column": 117761, + "rms_error": 0.061515625957307814, + "maximum_absolute_error": 0.1875 + }, + { + "column": 24129, + "rms_error": 0.06147691693569196, + "maximum_absolute_error": 0.3125 + }, + { + "column": 43345, + "rms_error": 0.06129237335627735, + "maximum_absolute_error": 0.15625 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-reasoning-64", + "reference_sha256": "8f7df6375fb0a0f3a96fabe16ae7f798be48785534fcb5cc3218b8225279378e" + }, + { + "rows": 257, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.00890914680506212, + "maximum_absolute_error": 0.375, + "differences": [ + { + "row": 6, + "native_top": 9798, + "reference_top": 9743, + "native_maximum_set": [ + 9798 + ], + "reference_maximum_set": [ + 9743, + 9798 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 102, + "rms_error": 0.12569535229309456, + "maximum_absolute_error": 0.375 + }, + { + "row": 141, + "rms_error": 0.12195062473144336, + "maximum_absolute_error": 0.25 + }, + { + "row": 51, + "rms_error": 0.11861646575515228, + "maximum_absolute_error": 0.34375 + }, + { + "row": 10, + "rms_error": 0.10028614776804698, + "maximum_absolute_error": 0.21875 + }, + { + "row": 121, + "rms_error": 0.09380736244683188, + "maximum_absolute_error": 0.25 + }, + { + "row": 219, + "rms_error": 0.09166578124562276, + "maximum_absolute_error": 0.25 + }, + { + "row": 57, + "rms_error": 0.0861529837832831, + "maximum_absolute_error": 0.21875 + }, + { + "row": 96, + "rms_error": 0.07888112333375753, + "maximum_absolute_error": 0.287109375 + }, + { + "row": 133, + "rms_error": 0.07786581656174694, + "maximum_absolute_error": 0.19140625 + }, + { + "row": 122, + "rms_error": 0.07066079887865517, + "maximum_absolute_error": 0.20703125 + }, + { + "row": 87, + "rms_error": 0.0700584951337384, + "maximum_absolute_error": 0.1875 + }, + { + "row": 77, + "rms_error": 0.06858409301583214, + "maximum_absolute_error": 0.22265625 + }, + { + "row": 174, + "rms_error": 0.06369936770618398, + "maximum_absolute_error": 0.2109375 + }, + { + "row": 109, + "rms_error": 0.06126578238264121, + "maximum_absolute_error": 0.1875 + }, + { + "row": 148, + "rms_error": 0.06105379414796407, + "maximum_absolute_error": 0.25 + }, + { + "row": 107, + "rms_error": 0.06088625375806041, + "maximum_absolute_error": 0.232421875 + }, + { + "row": 159, + "rms_error": 0.060035975951836636, + "maximum_absolute_error": 0.34375 + }, + { + "row": 228, + "rms_error": 0.05946765702581071, + "maximum_absolute_error": 0.171875 + }, + { + "row": 180, + "rms_error": 0.05838313205777614, + "maximum_absolute_error": 0.1875 + }, + { + "row": 146, + "rms_error": 0.05802808873794389, + "maximum_absolute_error": 0.3125 + } + ], + "worst_columns": [ + { + "column": 1008, + "rms_error": 0.06943595438926299, + "maximum_absolute_error": 0.3125 + }, + { + "column": 1703, + "rms_error": 0.0689546629516174, + "maximum_absolute_error": 0.34375 + }, + { + "column": 677, + "rms_error": 0.06758341579884398, + "maximum_absolute_error": 0.25 + }, + { + "column": 10625, + "rms_error": 0.06617273514319813, + "maximum_absolute_error": 0.2578125 + }, + { + "column": 334, + "rms_error": 0.06563620020264926, + "maximum_absolute_error": 0.28125 + }, + { + "column": 1840, + "rms_error": 0.06536974854013013, + "maximum_absolute_error": 0.3125 + }, + { + "column": 229, + "rms_error": 0.06493757056972531, + "maximum_absolute_error": 0.1875 + }, + { + "column": 10494, + "rms_error": 0.0645074011981935, + "maximum_absolute_error": 0.28125 + }, + { + "column": 3498, + "rms_error": 0.0636772495717454, + "maximum_absolute_error": 0.25 + }, + { + "column": 9798, + "rms_error": 0.06304420037383449, + "maximum_absolute_error": 0.234375 + }, + { + "column": 50474, + "rms_error": 0.06263534483934208, + "maximum_absolute_error": 0.21875 + }, + { + "column": 3431, + "rms_error": 0.06230194018215942, + "maximum_absolute_error": 0.25 + }, + { + "column": 3464, + "rms_error": 0.06201285570379185, + "maximum_absolute_error": 0.21875 + }, + { + "column": 4475, + "rms_error": 0.06195830330667422, + "maximum_absolute_error": 0.21875 + }, + { + "column": 405, + "rms_error": 0.06172813829250489, + "maximum_absolute_error": 0.28125 + }, + { + "column": 278, + "rms_error": 0.06139696424144821, + "maximum_absolute_error": 0.205810546875 + }, + { + "column": 25248, + "rms_error": 0.061211164521044456, + "maximum_absolute_error": 0.25 + }, + { + "column": 20, + "rms_error": 0.06067311917866256, + "maximum_absolute_error": 0.1875 + }, + { + "column": 417, + "rms_error": 0.059608407993605596, + "maximum_absolute_error": 0.375 + }, + { + "column": 25234, + "rms_error": 0.05953700598964491, + "maximum_absolute_error": 0.21875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-reasoning-257", + "reference_sha256": "76598d5e25b26caa3a49d87ab324926e3acf6714bebc1d049b215aabf0b3edd7", + "native_logits_sha256": "9be9b95af0ff0c00697f8737ce9271ccac0e6dcd519b56d49f3666623ae6fd2c" + }, + { + "rows": 513, + "vocabulary": 128000, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "relative_rms": 0.009213169639915648, + "maximum_absolute_error": 0.383544921875, + "differences": [ + { + "row": 6, + "native_top": 9798, + "reference_top": 9743, + "native_maximum_set": [ + 9798 + ], + "reference_maximum_set": [ + 9743, + 9798 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0625, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.0625, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 336, + "rms_error": 0.1881454590456434, + "maximum_absolute_error": 0.383544921875 + }, + { + "row": 102, + "rms_error": 0.12569535229309456, + "maximum_absolute_error": 0.375 + }, + { + "row": 141, + "rms_error": 0.12195062473144336, + "maximum_absolute_error": 0.25 + }, + { + "row": 51, + "rms_error": 0.11861646575515228, + "maximum_absolute_error": 0.34375 + }, + { + "row": 258, + "rms_error": 0.11009589234272762, + "maximum_absolute_error": 0.25 + }, + { + "row": 472, + "rms_error": 0.10046899842055106, + "maximum_absolute_error": 0.234375 + }, + { + "row": 10, + "rms_error": 0.10028614776804698, + "maximum_absolute_error": 0.21875 + }, + { + "row": 297, + "rms_error": 0.09552272607184104, + "maximum_absolute_error": 0.25 + }, + { + "row": 121, + "rms_error": 0.09380736244683188, + "maximum_absolute_error": 0.25 + }, + { + "row": 219, + "rms_error": 0.09166578124562276, + "maximum_absolute_error": 0.25 + }, + { + "row": 282, + "rms_error": 0.09102058524587438, + "maximum_absolute_error": 0.25 + }, + { + "row": 409, + "rms_error": 0.08696751249564189, + "maximum_absolute_error": 0.25 + }, + { + "row": 453, + "rms_error": 0.08674642545604302, + "maximum_absolute_error": 0.3125 + }, + { + "row": 57, + "rms_error": 0.0861529837832831, + "maximum_absolute_error": 0.21875 + }, + { + "row": 96, + "rms_error": 0.07888112333375753, + "maximum_absolute_error": 0.287109375 + }, + { + "row": 341, + "rms_error": 0.07793201637777376, + "maximum_absolute_error": 0.2265625 + }, + { + "row": 133, + "rms_error": 0.07786581656174694, + "maximum_absolute_error": 0.19140625 + }, + { + "row": 398, + "rms_error": 0.0764737367001577, + "maximum_absolute_error": 0.1875 + }, + { + "row": 360, + "rms_error": 0.07402143924854386, + "maximum_absolute_error": 0.25 + }, + { + "row": 473, + "rms_error": 0.07355884307503846, + "maximum_absolute_error": 0.1875 + } + ], + "worst_columns": [ + { + "column": 10625, + "rms_error": 0.06695082098979481, + "maximum_absolute_error": 0.2578125 + }, + { + "column": 50474, + "rms_error": 0.06628851063006382, + "maximum_absolute_error": 0.34375 + }, + { + "column": 1008, + "rms_error": 0.06555307895250304, + "maximum_absolute_error": 0.3125 + }, + { + "column": 229, + "rms_error": 0.06511421706049732, + "maximum_absolute_error": 0.3125 + }, + { + "column": 334, + "rms_error": 0.06446316244476662, + "maximum_absolute_error": 0.28125 + }, + { + "column": 3498, + "rms_error": 0.06416930226123667, + "maximum_absolute_error": 0.3125 + }, + { + "column": 1394, + "rms_error": 0.06413130019571951, + "maximum_absolute_error": 0.3125 + }, + { + "column": 677, + "rms_error": 0.0639696526546457, + "maximum_absolute_error": 0.25 + }, + { + "column": 278, + "rms_error": 0.06311357950539488, + "maximum_absolute_error": 0.2734375 + }, + { + "column": 10494, + "rms_error": 0.06296311952457531, + "maximum_absolute_error": 0.28125 + }, + { + "column": 32564, + "rms_error": 0.06291107980700647, + "maximum_absolute_error": 0.34375 + }, + { + "column": 3464, + "rms_error": 0.06151705018559607, + "maximum_absolute_error": 0.3125 + }, + { + "column": 25234, + "rms_error": 0.061327690045660375, + "maximum_absolute_error": 0.296875 + }, + { + "column": 1703, + "rms_error": 0.061288737510933354, + "maximum_absolute_error": 0.34375 + }, + { + "column": 1840, + "rms_error": 0.06109651463987475, + "maximum_absolute_error": 0.3125 + }, + { + "column": 405, + "rms_error": 0.06102842083328508, + "maximum_absolute_error": 0.28125 + }, + { + "column": 7161, + "rms_error": 0.060976322209715904, + "maximum_absolute_error": 0.2421875 + }, + { + "column": 4475, + "rms_error": 0.06074095619198881, + "maximum_absolute_error": 0.25 + }, + { + "column": 3431, + "rms_error": 0.06038757000041191, + "maximum_absolute_error": 0.25 + }, + { + "column": 22, + "rms_error": 0.06027237557813311, + "maximum_absolute_error": 0.19921875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-reasoning-513", + "reference_sha256": "bcd07462fa0dcdf7ff8eeb62e056046efa5f1a53c759fba11637eb80700dc2bd", + "native_logits_sha256": "049ebf779f8b00545a73253fb30fcd55cb1343321cb2f035fd5ad725153ddb66" + } + ], + "resolved_case_view": { + "cells": 17, + "rows": 5581, + "top1_differences": 19, + "disjoint_maximum_sets": 0, + "per_cell": [ + { + "cell": "1.7B-6-prose-257", + "rows": 257, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "1.7B-6-prose-513", + "rows": 481, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "1.7B-8-code-513", + "rows": 513, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-4-code-513", + "rows": 513, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-4-reasoning-257", + "rows": 257, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-4-reasoning-513", + "rows": 513, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-6-code-257", + "rows": 257, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-6-code-513", + "rows": 513, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-6-code-64", + "rows": 64, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-6-prose-257", + "rows": 257, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-6-prose-513", + "rows": 481, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-6-prose-64", + "rows": 64, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-6-reasoning-257", + "rows": 257, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane, RMS norm and seven-tap convolution reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-6-reasoning-513", + "rows": 513, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane, RMS norm and seven-tap convolution reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-6-reasoning-64", + "rows": 64, + "top1_differences": 1, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane, RMS norm and seven-tap convolution reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-8-prose-64", + "rows": 64, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + }, + { + "cell": "VL-2B-8-reasoning-513", + "rows": 513, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "rounding_sites": "FP32 affine lane reductions; remaining operations retain FP64" + } + ] + }, + "accepted_threshold": null, + "fidelity_pass": null +} diff --git a/docs/receipts/lfm3/rounding-reproduction.json b/docs/receipts/lfm3/rounding-reproduction.json new file mode 100644 index 0000000000..4774101681 --- /dev/null +++ b/docs/receipts/lfm3/rounding-reproduction.json @@ -0,0 +1,604 @@ +{ + "scope": "Independent whole-model FP64 calculation with production FP32 affine projection round sites; remaining operations retain FP64 arithmetic", + "accepted_threshold": null, + "fidelity_pass": null, + "cases": [ + { + "rows": 257, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.008228574718582158, + "maximum_absolute_error": 0.40234375, + "differences": [], + "worst_rows": [ + { + "row": 9, + "rms_error": 0.11675100600170359, + "maximum_absolute_error": 0.40234375 + }, + { + "row": 72, + "rms_error": 0.09182025948456946, + "maximum_absolute_error": 0.34375 + }, + { + "row": 30, + "rms_error": 0.08301323679998208, + "maximum_absolute_error": 0.2890625 + }, + { + "row": 34, + "rms_error": 0.08161881181226428, + "maximum_absolute_error": 0.234375 + }, + { + "row": 142, + "rms_error": 0.07404224594098688, + "maximum_absolute_error": 0.1875 + }, + { + "row": 250, + "rms_error": 0.06245770977221806, + "maximum_absolute_error": 0.171875 + }, + { + "row": 196, + "rms_error": 0.05974744907730566, + "maximum_absolute_error": 0.15625 + }, + { + "row": 181, + "rms_error": 0.05968768378833964, + "maximum_absolute_error": 0.15625 + }, + { + "row": 21, + "rms_error": 0.059383517969852855, + "maximum_absolute_error": 0.21875 + }, + { + "row": 141, + "rms_error": 0.05867767050374448, + "maximum_absolute_error": 0.203125 + }, + { + "row": 189, + "rms_error": 0.05833769401635122, + "maximum_absolute_error": 0.1875 + }, + { + "row": 29, + "rms_error": 0.05833556144228005, + "maximum_absolute_error": 0.21875 + }, + { + "row": 115, + "rms_error": 0.057935115778285945, + "maximum_absolute_error": 0.1640625 + }, + { + "row": 129, + "rms_error": 0.057801159107845855, + "maximum_absolute_error": 0.171875 + }, + { + "row": 106, + "rms_error": 0.05778176406071792, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 107, + "rms_error": 0.05745385391010654, + "maximum_absolute_error": 0.25 + }, + { + "row": 63, + "rms_error": 0.05726298677642242, + "maximum_absolute_error": 0.25 + }, + { + "row": 85, + "rms_error": 0.05675869907432556, + "maximum_absolute_error": 0.203125 + }, + { + "row": 150, + "rms_error": 0.05647164936702668, + "maximum_absolute_error": 0.21875 + }, + { + "row": 125, + "rms_error": 0.056325135119553775, + "maximum_absolute_error": 0.15625 + } + ], + "worst_columns": [ + { + "column": 10625, + "rms_error": 0.0719168286234429, + "maximum_absolute_error": 0.3125 + }, + { + "column": 16420, + "rms_error": 0.06935220883801685, + "maximum_absolute_error": 0.3125 + }, + { + "column": 7161, + "rms_error": 0.06872535279635689, + "maximum_absolute_error": 0.28125 + }, + { + "column": 334, + "rms_error": 0.06852811266791806, + "maximum_absolute_error": 0.29296875 + }, + { + "column": 1840, + "rms_error": 0.06808063619059351, + "maximum_absolute_error": 0.34375 + }, + { + "column": 50474, + "rms_error": 0.06801655566196134, + "maximum_absolute_error": 0.34375 + }, + { + "column": 229, + "rms_error": 0.06790218938914094, + "maximum_absolute_error": 0.28125 + }, + { + "column": 10494, + "rms_error": 0.06760618030686806, + "maximum_absolute_error": 0.296875 + }, + { + "column": 3464, + "rms_error": 0.06581870418435679, + "maximum_absolute_error": 0.28125 + }, + { + "column": 278, + "rms_error": 0.06574005733457104, + "maximum_absolute_error": 0.359375 + }, + { + "column": 677, + "rms_error": 0.0644634874054111, + "maximum_absolute_error": 0.21875 + }, + { + "column": 2925, + "rms_error": 0.06352312981336071, + "maximum_absolute_error": 0.28125 + }, + { + "column": 90208, + "rms_error": 0.06342204452206861, + "maximum_absolute_error": 0.34375 + }, + { + "column": 3431, + "rms_error": 0.0632776334042306, + "maximum_absolute_error": 0.265625 + }, + { + "column": 3300, + "rms_error": 0.0631922709910923, + "maximum_absolute_error": 0.203125 + }, + { + "column": 3498, + "rms_error": 0.06288495654896703, + "maximum_absolute_error": 0.21875 + }, + { + "column": 32564, + "rms_error": 0.062457497914349076, + "maximum_absolute_error": 0.28125 + }, + { + "column": 405, + "rms_error": 0.06221173376451575, + "maximum_absolute_error": 0.296875 + }, + { + "column": 20, + "rms_error": 0.06212506474275177, + "maximum_absolute_error": 0.21875 + }, + { + "column": 91312, + "rms_error": 0.061823161313847616, + "maximum_absolute_error": 0.21875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-4-reasoning-257", + "original_ideal_fp64_differences": [ + { + "row": 17, + "native_top": 296, + "reference_top": 405, + "native_maximum_set": [ + 296 + ], + "reference_maximum_set": [ + 405 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.125, + "native_reference_contender_gap": 0.1875, + "reference_native_contender_gap": 0.125, + "rounded_reference_top": 296, + "rounded_reference_maximum_set": [ + 296 + ], + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": 0.0625 + }, + { + "row": 18, + "native_top": 22, + "reference_top": 1018, + "native_maximum_set": [ + 22, + 1018 + ], + "reference_maximum_set": [ + 1018 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625, + "rounded_reference_top": 22, + "rounded_reference_maximum_set": [ + 22 + ], + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": 0.0625 + } + ], + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "118440ccd7fc07a502485a3266a2878ec815f316bcfa9dddd859e5d324fb8654", + "ideal_reference_logits_sha256": "eaad83f4ec389f5c731c0d3422f2af47f218f8cdb220a6d0b752641c8f8f9bfe", + "rounded_reference_logits_sha256": "b59c43083bfc6008233e7e249ced2f589ace92e1461e95c65300141aea14ab2b" + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 0, + "relative_rms": 0.013509190417066086, + "maximum_absolute_error": 0.625, + "differences": [ + { + "row": 13, + "native_top": 22, + "reference_top": 63908, + "native_maximum_set": [ + 22, + 63908 + ], + "reference_maximum_set": [ + 63908 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 22, + "native_top": 16033, + "reference_top": 91, + "native_maximum_set": [ + 16033 + ], + "reference_maximum_set": [ + 91, + 16033 + ], + "maximum_sets_overlap": true, + "native_margin": 0.125, + "reference_margin": 0.0, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.0 + } + ], + "worst_rows": [ + { + "row": 57, + "rms_error": 0.09000787683806573, + "maximum_absolute_error": 0.625 + }, + { + "row": 10, + "rms_error": 0.08800637679348153, + "maximum_absolute_error": 0.234375 + }, + { + "row": 54, + "rms_error": 0.07760607841137634, + "maximum_absolute_error": 0.375 + }, + { + "row": 36, + "rms_error": 0.058355442854574543, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 53, + "rms_error": 0.057159352869539715, + "maximum_absolute_error": 0.21875 + }, + { + "row": 20, + "rms_error": 0.055633565618538675, + "maximum_absolute_error": 0.296875 + }, + { + "row": 39, + "rms_error": 0.053347736683817384, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 13, + "rms_error": 0.052123672609206625, + "maximum_absolute_error": 0.21875 + }, + { + "row": 8, + "rms_error": 0.05116690334326474, + "maximum_absolute_error": 0.25 + }, + { + "row": 2, + "rms_error": 0.05008304981339593, + "maximum_absolute_error": 0.125 + }, + { + "row": 52, + "rms_error": 0.04974962183100654, + "maximum_absolute_error": 0.1875 + }, + { + "row": 62, + "rms_error": 0.04902776384179979, + "maximum_absolute_error": 0.21875 + }, + { + "row": 55, + "rms_error": 0.047242131140363866, + "maximum_absolute_error": 0.21875 + }, + { + "row": 23, + "rms_error": 0.044142037283058606, + "maximum_absolute_error": 0.1875 + }, + { + "row": 11, + "rms_error": 0.043190213412404824, + "maximum_absolute_error": 0.25 + }, + { + "row": 22, + "rms_error": 0.04095067360254933, + "maximum_absolute_error": 0.3125 + }, + { + "row": 38, + "rms_error": 0.04044440498995601, + "maximum_absolute_error": 0.234375 + }, + { + "row": 30, + "rms_error": 0.04023781052786102, + "maximum_absolute_error": 0.1953125 + }, + { + "row": 50, + "rms_error": 0.0400162730201504, + "maximum_absolute_error": 0.171875 + }, + { + "row": 26, + "rms_error": 0.03986160470382173, + "maximum_absolute_error": 0.25 + } + ], + "worst_columns": [ + { + "column": 538, + "rms_error": 0.09985948465502727, + "maximum_absolute_error": 0.625 + }, + { + "column": 33237, + "rms_error": 0.09714630618708237, + "maximum_absolute_error": 0.375 + }, + { + "column": 723, + "rms_error": 0.09163830636319745, + "maximum_absolute_error": 0.4375 + }, + { + "column": 16033, + "rms_error": 0.0907320804033186, + "maximum_absolute_error": 0.34375 + }, + { + "column": 6986, + "rms_error": 0.0893078048926329, + "maximum_absolute_error": 0.28125 + }, + { + "column": 30270, + "rms_error": 0.08854197407562756, + "maximum_absolute_error": 0.34375 + }, + { + "column": 9700, + "rms_error": 0.0861797386586336, + "maximum_absolute_error": 0.3125 + }, + { + "column": 61089, + "rms_error": 0.08516518525177089, + "maximum_absolute_error": 0.25 + }, + { + "column": 116016, + "rms_error": 0.08419055912772351, + "maximum_absolute_error": 0.3125 + }, + { + "column": 29956, + "rms_error": 0.08386143846355991, + "maximum_absolute_error": 0.28125 + }, + { + "column": 39382, + "rms_error": 0.0833193654285085, + "maximum_absolute_error": 0.328125 + }, + { + "column": 274, + "rms_error": 0.08328050464397146, + "maximum_absolute_error": 0.46875 + }, + { + "column": 6907, + "rms_error": 0.08258207761026035, + "maximum_absolute_error": 0.552734375 + }, + { + "column": 42847, + "rms_error": 0.08252694023323655, + "maximum_absolute_error": 0.296875 + }, + { + "column": 1583, + "rms_error": 0.08156546550782054, + "maximum_absolute_error": 0.25 + }, + { + "column": 295, + "rms_error": 0.08148593392714178, + "maximum_absolute_error": 0.4375 + }, + { + "column": 717, + "rms_error": 0.08125915112683267, + "maximum_absolute_error": 0.515625 + }, + { + "column": 93538, + "rms_error": 0.08125813356366732, + "maximum_absolute_error": 0.46875 + }, + { + "column": 2741, + "rms_error": 0.08083424732039579, + "maximum_absolute_error": 0.45703125 + }, + { + "column": 346, + "rms_error": 0.08050453816084938, + "maximum_absolute_error": 0.5 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-code-64", + "original_ideal_fp64_differences": [ + { + "row": 13, + "native_top": 22, + "reference_top": 63908, + "native_maximum_set": [ + 22, + 63908 + ], + "reference_maximum_set": [ + 63908 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625, + "rounded_reference_top": 63908, + "rounded_reference_maximum_set": [ + 63908 + ], + "rounded_reference_margin": 0.0625, + "rounded_reference_native_contender_gap": -0.0625 + }, + { + "row": 22, + "native_top": 16033, + "reference_top": 91, + "native_maximum_set": [ + 16033 + ], + "reference_maximum_set": [ + 91 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.1875, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.1875, + "rounded_reference_top": 91, + "rounded_reference_maximum_set": [ + 91, + 16033 + ], + "rounded_reference_margin": 0.0, + "rounded_reference_native_contender_gap": 0.0 + } + ], + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655", + "native_logits_sha256": "02225f1c8c7f5de05ebd44bd49948ad000aac040f3d6025d6655a9772ed08f9b", + "ideal_reference_logits_sha256": "d16f06c7813c231e4f3b7172456959da94d32adf024381666104fa28c0d3a859", + "rounded_reference_logits_sha256": "3f061bbfb2d31d235f0c4c5681e170f1bb2fe5f928ff805bf768d7727787e614" + } + ], + "first_projection_raw_replay": { + "shape": [ + 257, + 6144 + ], + "projection": "language_model.model.layers.0.conv.in_proj", + "scope": "Projection rounding replay; not a whole-model fidelity pass", + "affine_fusion": "Explicit CPU fmaf", + "accepted_threshold": null, + "raw_equal": 1579008, + "values": 1579008, + "bf16_equal": 1579008, + "maximum_absolute_error": 0.0 + }, + "first_convolution_same_input_rounding_replay": { + "conv": { + "raw_fp32_sum": true, + "bf16_equal": 526336, + "values": 526336, + "max_error": 0.0 + } + }, + "optional_accelerator": { + "source": "tools/lfm3/row_projection_replay.c", + "compile": "cc -std=c11 -O3 -ffp-contract=off -shared -fPIC tools/lfm3/row_projection_replay.c -o row-replay.so", + "equivalence": "All 1579008 captured pre-store FP32 outputs equal both NumPy and explicit C fmaf replays" + } +} diff --git a/docs/receipts/lfm3/scratch-reuse.json b/docs/receipts/lfm3/scratch-reuse.json new file mode 100644 index 0000000000..3dba7cf53c --- /dev/null +++ b/docs/receipts/lfm3/scratch-reuse.json @@ -0,0 +1,173 @@ +{ + "scope": "Private one-allocation bcx/act alias; local M4 Pro text and VL 4-bit correctness only", + "runtime_change": { + "file": "zig/src/families/lfm3/state.zig", + "baseline_sha256": "1161aea9174fd6a6dbecc09bfd26565459c9d7b8d1674ca06c7c78c1b0265517", + "candidate_sha256": "8a7083b1c43c75803596d441f20c5cfefb00820738adbcc65ffc6b043e647e10", + "patch_sha256": "3748e191f85ff8871df3fe108a836764ba6b269276f937169a565aed65aabba6", + "source_base": "Private GPU proof predates lifecycle92c64e96f; final applied public-source host/build check below combines lifecycle plus the exact same state alias. GPU binaries and final public build binaries are distinct artifacts." + }, + "allocation": { + "method": "Same instrumentedMetal mock invokes actual baseline/candidateScratch.init withactualconfig.schedule,rows128;testingallocator checkslifetime. No mirroredallocationimplementation.", + "baseline": { + "owned_allocations": 34, + "requested_buffer_bytes": 63677056, + "joins": 34, + "leaves": 34, + "frees": 34, + "actual_exit": 0 + }, + "candidate": { + "owned_allocations": 33, + "requested_buffer_bytes": 62104192, + "joins": 33, + "leaves": 33, + "frees": 33, + "actual_exit": 0 + }, + "saved_bytes": 1572864, + "saved_allocations": 1, + "test_sources_sha256": { + "lfm3-scratch-alias-ownership-test.zig": "5bbd1fc7ae26ebcc3f72dfbdd1f243e84795f038c915abdd50bcc19f75022d2c", + "lfm3-scratch-alias-metal-mock.zig": "e4f52bcadfaf21d56116fe98d71f15d9033e718f0ecc28e82329f2cc4ba139c6" + }, + "limits": "Allocation counts are requested-buffer bytes from the instrumented host allocator; actual 4-bit GPU correctness is recorded below. No physical-footprint or speed claim." + }, + "saved_requested_buffer_bytes_at128rows": 1572864, + "build": { + "native_forward_exact": "65/65 steps succeeded", + "profile": "Build succeeded", + "clock_fields_discarded": true + }, + "binaries": { + "zig-out/bin/tf-lfm3-forward": "160b350b19fc5c8f9273cb23d4cb2de649a8cdb4eca22f8318686ef7beeab2d5", + "zig-out/bin/tf-lfm3-exact": "addf6e1bd5e1a27e58f2a98104dd083ed9debe01efd85f3c34f650cb07df0463", + "zig-out/bin/tf-lfm3-profile": "d06ef0507b33b1ae3900e67eccd648bc2e4553c87b3de32d4189e370ab8db1fb", + "zig-out/native/bin/tensorfold-native": "da3157a08b0b7899d4c37ed74b0251b112f6f09bd3108032baa4c9d5f9a67507" + }, + "models": [ + { + "model": "LFM3-1.7B-MLX-4bit", + "state_passed": 68, + "state_failed": 0, + "logit_bytes": 65792000, + "logits_current_public_exact": true, + "logits_sha256": "f1dbd42c2556fcb05031a00b209c61fadee45e74ee1ff94628d4802479fcb518" + }, + { + "model": "LFM3-VL-2B-MLX-4bit", + "state_passed": 68, + "state_failed": 0, + "logit_bytes": 65792000, + "logits_current_public_exact": true, + "logits_sha256": "a4dde0a00ee9acb294c2b5a791e7e73cd05c5d6904ffe171e439ad1b01e25c5d" + } + ], + "traces": [ + { + "event": "trace_complete", + "model": "LFM3-1.7B", + "files": 620, + "bytes": 68474208, + "exact": true + }, + { + "event": "trace_complete", + "model": "LFM3-VL-2B", + "files": 620, + "bytes": 68474208, + "exact": true + }, + { + "event": "image_complete", + "rows": 90, + "image_rows": 70, + "trace_files": 930, + "exact": true, + "decoder_logit_bytes": 23040000, + "trace_bytes": 186747840 + } + ], + "profile_identity": [ + { + "event": "profile_identity", + "model": "LFM3-1.7B", + "mode": "stage", + "plain_profile_current_public_final_logits_equal": true, + "timing_fields_discarded": true + }, + { + "event": "profile_identity", + "model": "LFM3-VL-2B", + "mode": "stage", + "plain_profile_current_public_final_logits_equal": true, + "timing_fields_discarded": true + } + ], + "lease": { + "path": "/tmp/tensorfold-model.lock", + "exclusive": true, + "same_alias_inode": true, + "single_model_admission_before_and_after_child": true, + "owned_children_exited_before_release": true, + "local_only": true + }, + "scope_limits": [ + "No 6/8-bit runs.", + "No speed, independent FP64 fidelity, or physical-footprint reduction claim.", + "Image test injects retained real-cat projected rows; vision tower itself is unchanged and was not rerun.", + "All timing fields from correctness tools are discarded." + ], + "evidence_sha256": { + "lfm3-scratch-alias-proof/LFM3-1.7B-MLX-4bit-state.log": "fca04ca0d1f4afdde25a037178a11177a62909f668eca86ed8923d47bc00c1f0", + "lfm3-scratch-alias-proof/events.json": "e6b6e0682e6badd3070ada9243bce39a7e3d24ed0eac87f949eb935a38476231", + "lfm3-scratch-alias-proof/LFM3-1.7B-MLX-4bit-combined-logits.log": "da6d8a43e75cd674009591a7a830d2907f6b8f558c0c494f40d1cc4178bd82dc", + "lfm3-scratch-alias-proof/complete.json": "ceb54aca0f65edf2a3ad310e66a956075b4c9881d5557bfca5f09ed5f29b7405", + "lfm3-scratch-alias-proof/LFM3-VL-2B-MLX-4bit-state.log": "e7ccee7df28d5c8188e225b2064bffb275ca94e835da1548dc912b0456ba16d5", + "lfm3-scratch-alias-proof/LFM3-VL-2B-MLX-4bit-public-logits.log": "da6d8a43e75cd674009591a7a830d2907f6b8f558c0c494f40d1cc4178bd82dc", + "lfm3-scratch-alias-proof/records.json": "38b3e5edcfa2b6db327578ab4a19d1064ab105d126d15c4526b1f44201185309", + "lfm3-scratch-alias-proof/LFM3-1.7B-old-trace.log": "790d8930bff73e6e081f6e535f57733e9ea8b951dc3cb473c9a309bba80cb2ba", + "lfm3-scratch-alias-proof/LFM3-1.7B-MLX-4bit-public-logits.log": "da6d8a43e75cd674009591a7a830d2907f6b8f558c0c494f40d1cc4178bd82dc", + "lfm3-scratch-alias-proof/LFM3-1.7B-new-trace.log": "790d8930bff73e6e081f6e535f57733e9ea8b951dc3cb473c9a309bba80cb2ba", + "lfm3-scratch-alias-proof/LFM3-VL-2B-new-trace.log": "790d8930bff73e6e081f6e535f57733e9ea8b951dc3cb473c9a309bba80cb2ba", + "lfm3-scratch-alias-proof/LFM3-VL-2B-MLX-4bit-combined-logits.log": "da6d8a43e75cd674009591a7a830d2907f6b8f558c0c494f40d1cc4178bd82dc", + "lfm3-scratch-alias-proof/LFM3-VL-2B-old-trace.log": "790d8930bff73e6e081f6e535f57733e9ea8b951dc3cb473c9a309bba80cb2ba", + "lfm3-scratch-alias-image-proof/LFM3-1.7B-profile.log": "d930bc329a5754114b8de4a5ef47a0674fcdfacc593b37007d7a073de2b6e1f4", + "lfm3-scratch-alias-image-proof/image-old.log": "aafffce9158d8f03ef1a53cabcbc801a169cf1f973d8a2faaf5cee07d5787444", + "lfm3-scratch-alias-image-proof/events.json": "4f708b7320dd227d9f17495f1f97d68ef7bab04592f1e58ad309d4b7acb28d57", + "lfm3-scratch-alias-image-proof/LFM3-VL-2B-profile.json": "382ada3c09848ebc0eb8c2d2db6c53eda087863ff9281b8daf1f7dd69c11b121", + "lfm3-scratch-alias-image-proof/LFM3-1.7B-profile.json": "2b29b7e287e1b5f917509f7a6e3bcf038c2c8de2e6fb634afe5446e63bdc6380", + "lfm3-scratch-alias-image-proof/LFM3-VL-2B-profile.log": "d930bc329a5754114b8de4a5ef47a0674fcdfacc593b37007d7a073de2b6e1f4", + "lfm3-scratch-alias-image-proof/image-new.log": "aafffce9158d8f03ef1a53cabcbc801a169cf1f973d8a2faaf5cee07d5787444" + }, + "inputs": { + "code-257.npy": "96859365056d513a414e6eeef9d4a85bf72c3f9630d3e13a9ea0f57dd81b8bea", + "tokens.npy": "ab18390677209ae355cbc0d64e33f94856dc70264f01c8b94a18df310a7712e8", + "lfm3-real-cat-tokens.npy": "54c970db8c6ba82895cb2f18a48d397ea568ae885241129e9c5ee7dbbc91a24d", + "lfm3-real-cat-native.bf16": "ab2ac7db759e5c4a32fe7d60680ec47f7d1cbc5938fa7d8091bf4756bfeea4fc" + }, + "combined_public_build": { + "base_commit": "92c64e96f56d1a7a68f8476362696b4419b973b9", + "command": "ZIG_GLOBAL_CACHE_DIR=$WRITABLE_CACHE zig build test-lfm3-host native tf-lfm3-forward tf-lfm3-exact --cache-dir $PUBLIC_CACHE --summary all", + "actual_exit": 0, + "steps_passed": 77, + "steps_total": 77, + "host_tests_passed": 44, + "host_tests_total": 44, + "jpeg_limitation": "Pinned libjpeg-turbo3.1.4.1 unavailable; JPEG pixel cases skipped, warning retained.", + "fmt_check": true, + "diff_check": true, + "lean_problems": 428, + "lean_new_problems": 0, + "state_source_sha256": "8a7083b1c43c75803596d441f20c5cfefb00820738adbcc65ffc6b043e647e10", + "backend_source_sha256": "4ed5d33b869dc4be0d82d406cff4abc0f65a63687c45d8b3aaa61e26361db233", + "binaries_sha256": { + "zig-out/native/bin/tensorfold-native": "aae2de20cec9698afa9c108a4f7de27b83e5695b61f45cbb5e3dc21c34f6fbb3", + "zig-out/bin/tf-lfm3-forward": "7b81ba7487d3c07625ea3c6be45e0e4ae475373b6797af3ac6aae0717aa78b14", + "zig-out/bin/tf-lfm3-exact": "6dda9934d296db6bc76147fba113848fdf22df0ae45d1ead56f827cc00eb290b" + }, + "build_log_sha256": "2bacc236d342187ff20fdc4e1ce7a809e969af40da610c544ccfd36b4d2b4ddd", + "lean_log_sha256": "2aceae81fc338ffce3e33e5ec0a51b733e33eee717e102384f750bf41e808a01", + "gpu_rerun": false + } +} diff --git a/docs/receipts/lfm3/served-checks.json b/docs/receipts/lfm3/served-checks.json new file mode 100644 index 0000000000..6b93572c11 --- /dev/null +++ b/docs/receipts/lfm3/served-checks.json @@ -0,0 +1,10807 @@ +{ + "scope": "M4 Pro frozen benchmark candidate and M5 Ultra final-source candidate; generated text, transient request IDs, log paths and local commands omitted", + "cells": [ + { + "checkpoint": "LFM3-1.7B-MLX-4bit", + "native_binary_sha256": "5c8a04d4af599879797f4f5bed8e00ab7d9d70732e698bd2a1ec2a01ec762e09", + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + } + ], + "platform": "M4 Pro, macOS 26.5.1" + }, + { + "checkpoint": "LFM3-1.7B-MLX-6bit", + "native_binary_sha256": "5c8a04d4af599879797f4f5bed8e00ab7d9d70732e698bd2a1ec2a01ec762e09", + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + } + ], + "platform": "M4 Pro, macOS 26.5.1" + }, + { + "checkpoint": "LFM3-1.7B-MLX-8bit", + "native_binary_sha256": "5c8a04d4af599879797f4f5bed8e00ab7d9d70732e698bd2a1ec2a01ec762e09", + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + } + ], + "platform": "M4 Pro, macOS 26.5.1" + }, + { + "checkpoint": "LFM3-VL-2B-MLX-4bit", + "native_binary_sha256": "5c8a04d4af599879797f4f5bed8e00ab7d9d70732e698bd2a1ec2a01ec762e09", + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + }, + { + "name": "blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef" + } + }, + { + "name": "red_blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40" + } + }, + { + "name": "blue_red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + }, + { + "name": "red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + }, + { + "name": "blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef" + } + }, + { + "name": "blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef" + } + }, + { + "name": "red_blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40" + } + }, + { + "name": "red_blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40" + } + }, + { + "name": "blue_red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf" + } + }, + { + "name": "blue_red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red" + ], + "observed": [ + "red", + "red", + "red" + ] + } + }, + { + "name": "blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue" + ], + "observed": [ + "blue", + "blue", + "blue" + ] + } + }, + { + "name": "red_blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red", + "blue" + ], + "observed": [ + "red", + "blue" + ] + } + }, + { + "name": "blue_red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue", + "red" + ], + "observed": [ + "blue", + "red" + ] + } + }, + { + "name": "images/concurrent_equals_solos", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + }, + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf" + } + ] + }, + { + "name": "images/corrupt_png_rejected", + "status": "passed", + "evidence": { + "status": 400, + "complete": false + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "red/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 2, + "server_disconnections_after": 3, + "server_confirmed_cancel": true + } + }, + { + "name": "red/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + }, + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + } + ], + "platform": "M4 Pro, macOS 26.5.1" + }, + { + "checkpoint": "LFM3-VL-2B-MLX-6bit", + "native_binary_sha256": "5c8a04d4af599879797f4f5bed8e00ab7d9d70732e698bd2a1ec2a01ec762e09", + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + }, + { + "name": "red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + }, + { + "name": "blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0" + } + }, + { + "name": "red_blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5" + } + }, + { + "name": "blue_red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + }, + { + "name": "red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + }, + { + "name": "red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + }, + { + "name": "blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0" + } + }, + { + "name": "blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0" + } + }, + { + "name": "red_blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5" + } + }, + { + "name": "red_blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5" + } + }, + { + "name": "blue_red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603" + } + }, + { + "name": "blue_red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + ] + }, + { + "name": "red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red" + ], + "observed": [ + "red", + "red" + ] + } + }, + { + "name": "blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue" + ], + "observed": [ + "blue", + "blue" + ] + } + }, + { + "name": "red_blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red", + "blue" + ], + "observed": [ + "red", + "blue" + ] + } + }, + { + "name": "blue_red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue", + "red" + ], + "observed": [ + "blue", + "red", + "blue" + ] + } + }, + { + "name": "images/concurrent_equals_solos", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + }, + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603" + } + ] + }, + { + "name": "images/corrupt_png_rejected", + "status": "passed", + "evidence": { + "status": 400, + "complete": false + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "red/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 2, + "server_disconnections_after": 3, + "server_confirmed_cancel": true + } + }, + { + "name": "red/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + }, + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + } + ], + "platform": "M4 Pro, macOS 26.5.1" + }, + { + "checkpoint": "LFM3-VL-2B-MLX-8bit", + "native_binary_sha256": "5c8a04d4af599879797f4f5bed8e00ab7d9d70732e698bd2a1ec2a01ec762e09", + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + }, + { + "name": "blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6" + } + }, + { + "name": "red_blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f" + } + }, + { + "name": "blue_red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + }, + { + "name": "red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + }, + { + "name": "blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6" + } + }, + { + "name": "blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6" + } + }, + { + "name": "red_blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f" + } + }, + { + "name": "red_blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f" + } + }, + { + "name": "blue_red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005" + } + }, + { + "name": "blue_red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red" + ], + "observed": [ + "red", + "red" + ] + } + }, + { + "name": "blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue" + ], + "observed": [ + "blue", + "blue", + "blue" + ] + } + }, + { + "name": "red_blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red", + "blue" + ], + "observed": [ + "red", + "blue" + ] + } + }, + { + "name": "blue_red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue", + "red" + ], + "observed": [ + "blue", + "red" + ] + } + }, + { + "name": "images/concurrent_equals_solos", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + }, + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005" + } + ] + }, + { + "name": "images/corrupt_png_rejected", + "status": "passed", + "evidence": { + "status": 400, + "complete": false + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "red/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 2, + "server_disconnections_after": 3, + "server_confirmed_cancel": true + } + }, + { + "name": "red/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + }, + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + } + ], + "platform": "M4 Pro, macOS 26.5.1" + }, + { + "platform": "M5 Ultra, macOS 27.0", + "checkpoint": "LFM3-1.7B-MLX-4bit", + "native_binary_sha256": "251079309919aa9b21ca8cc6a89c58fc3fc9a9baf800fa5ce6a6af313a5b5f65", + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + } + ] + }, + { + "platform": "M5 Ultra, macOS 27.0", + "checkpoint": "LFM3-1.7B-MLX-6bit", + "native_binary_sha256": "251079309919aa9b21ca8cc6a89c58fc3fc9a9baf800fa5ce6a6af313a5b5f65", + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + } + ] + }, + { + "platform": "M5 Ultra, macOS 27.0", + "checkpoint": "LFM3-1.7B-MLX-8bit", + "native_binary_sha256": "251079309919aa9b21ca8cc6a89c58fc3fc9a9baf800fa5ce6a6af313a5b5f65", + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + } + ] + }, + { + "platform": "M5 Ultra, macOS 27.0", + "checkpoint": "LFM3-VL-2B-MLX-4bit", + "native_binary_sha256": "251079309919aa9b21ca8cc6a89c58fc3fc9a9baf800fa5ce6a6af313a5b5f65", + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + }, + { + "name": "blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef" + } + }, + { + "name": "red_blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40" + } + }, + { + "name": "blue_red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + }, + { + "name": "red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + }, + { + "name": "blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef" + } + }, + { + "name": "blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef" + } + }, + { + "name": "red_blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40" + } + }, + { + "name": "red_blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40" + } + }, + { + "name": "blue_red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf" + } + }, + { + "name": "blue_red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red" + ], + "observed": [ + "red", + "red", + "red" + ] + } + }, + { + "name": "blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue" + ], + "observed": [ + "blue", + "blue", + "blue" + ] + } + }, + { + "name": "red_blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red", + "blue" + ], + "observed": [ + "red", + "blue" + ] + } + }, + { + "name": "blue_red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue", + "red" + ], + "observed": [ + "blue", + "red" + ] + } + }, + { + "name": "images/concurrent_equals_solos", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + }, + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf" + } + ] + }, + { + "name": "images/corrupt_png_rejected", + "status": "passed", + "evidence": { + "status": 400, + "complete": false + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "red/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 2, + "server_disconnections_after": 3, + "server_confirmed_cancel": true + } + }, + { + "name": "red/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + }, + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + } + ] + }, + { + "platform": "M5 Ultra, macOS 27.0", + "checkpoint": "LFM3-VL-2B-MLX-6bit", + "native_binary_sha256": "251079309919aa9b21ca8cc6a89c58fc3fc9a9baf800fa5ce6a6af313a5b5f65", + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + }, + { + "name": "red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + }, + { + "name": "blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0" + } + }, + { + "name": "red_blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5" + } + }, + { + "name": "blue_red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + }, + { + "name": "red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + }, + { + "name": "red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + }, + { + "name": "blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0" + } + }, + { + "name": "blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0" + } + }, + { + "name": "red_blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5" + } + }, + { + "name": "red_blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5" + } + }, + { + "name": "blue_red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603" + } + }, + { + "name": "blue_red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + ] + }, + { + "name": "red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red" + ], + "observed": [ + "red", + "red" + ] + } + }, + { + "name": "blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue" + ], + "observed": [ + "blue", + "blue" + ] + } + }, + { + "name": "red_blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red", + "blue" + ], + "observed": [ + "red", + "blue" + ] + } + }, + { + "name": "blue_red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue", + "red" + ], + "observed": [ + "blue", + "red", + "blue" + ] + } + }, + { + "name": "images/concurrent_equals_solos", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + }, + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603" + } + ] + }, + { + "name": "images/corrupt_png_rejected", + "status": "passed", + "evidence": { + "status": 400, + "complete": false + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "red/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 2, + "server_disconnections_after": 3, + "server_confirmed_cancel": true + } + }, + { + "name": "red/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + }, + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + } + ] + }, + { + "platform": "M5 Ultra, macOS 27.0", + "checkpoint": "LFM3-VL-2B-MLX-8bit", + "native_binary_sha256": "251079309919aa9b21ca8cc6a89c58fc3fc9a9baf800fa5ce6a6af313a5b5f65", + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9, + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + }, + { + "name": "blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6" + } + }, + { + "name": "red_blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f" + } + }, + { + "name": "blue_red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + }, + { + "name": "red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + }, + { + "name": "blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6" + } + }, + { + "name": "blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6" + } + }, + { + "name": "red_blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f" + } + }, + { + "name": "red_blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f" + } + }, + { + "name": "blue_red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005" + } + }, + { + "name": "blue_red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red" + ], + "observed": [ + "red", + "red" + ] + } + }, + { + "name": "blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue" + ], + "observed": [ + "blue", + "blue", + "blue" + ] + } + }, + { + "name": "red_blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red", + "blue" + ], + "observed": [ + "red", + "blue" + ] + } + }, + { + "name": "blue_red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue", + "red" + ], + "observed": [ + "blue", + "red" + ] + } + }, + { + "name": "images/concurrent_equals_solos", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + }, + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005" + } + ] + }, + { + "name": "images/corrupt_png_rejected", + "status": "passed", + "evidence": { + "status": 400, + "complete": false + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "red/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 2, + "server_disconnections_after": 3, + "server_confirmed_cancel": true + } + }, + { + "name": "red/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + }, + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + } + ] + } + ] +} diff --git a/docs/receipts/lfm3/shared-attention.json b/docs/receipts/lfm3/shared-attention.json new file mode 100644 index 0000000000..abb0ab6ea4 --- /dev/null +++ b/docs/receipts/lfm3/shared-attention.json @@ -0,0 +1,5468 @@ +{ + "scope": "Shared-stream native Metal attention preserves serial attention bytes; this is not a tensor-unit prompt-attention qualification.", + "binary_sha256": "d1656249bdb581b147faacbd3b3c557c00b6c45b710eeddce517a67178b6ba2b", + "source_manifest": { + "zig/kernels/metal/lfm3/attention.metal": "cfd691baa70a30cd7629b1ac4c795d58ae2bad95aff07a2e6fb2347e1bf9b514", + "zig/src/families/lfm3/kernels.zig": "73406a56a0ad15700b9da797cab84fea75d0b94c0e58cd4d8abf7b84638048ec", + "zig/tests/contract/metal_lfm3_segments.zig": "036899bf4a5323ccce661386a35a9b3cd638f7a8c6a9c004b78094a9e0be4c24", + "tools/lfm3/serve_check.py": "87327328ef1a1fcc35536efa0c18829779ee278a4d325f2af69356bc68a71f5c" + }, + "source_manifest_scope": "Hashes identify reviewed shared-attention source and checker files; they are not a whole-tree build manifest.", + "operator_contract": { + "hardware": "M4 Pro, 48 GB, macOS 26.5.1", + "command": "tf-kernel-contract --only LFM3", + "exit_status": 0, + "summary": "SUMMARY 52 kernels: 52 pass gates 1-4, 0 fail, 0 known findings; 14 controls: 14 red as planted, 0 missed; 0 limit problems; gates 28006 ms, limits 107 ms", + "shared_operator_scope": "Full and sliding serial-to-shared byte replacement; real shader missing-tail control; existing FP64 and accumulation gates retained." + }, + "exactness": { + "hardware": "M4 Pro, 48 GB, macOS 26.5.1", + "checkpoint": "LFM3-1.7B-MLX-4bit", + "command": "tf-lfm3-exact MODEL_DIR", + "exit_status": 0, + "summary": "LFM3_EXACT mode=default passed=68 failed=0 skipped=0 prompt=5 windows=60 shared=2 snapshots=1 long=0 gpu_seconds=148.781", + "scope": "Includes two unequal streams and four unequal streams with 32/16 aggregate rows, partial keeps, idle stream, distinct cache capacities and state/logit equality against one-row execution. This run did not include --long." + }, + "served": { + "hardware": "M5 Ultra, 256 GiB, macOS 27.0 (26A428)", + "command": "python3 tools/lfm3/serve_check.py --binary NATIVE --model-dir MODEL_DIR --sampled --cancel --output RESULT.json", + "counts": { + "passed": 303, + "failed": 0, + "skipped": 0 + }, + "cells": [ + { + "model": "LFM3-1.7B-MLX-4bit", + "binary_sha256": "d1656249bdb581b147faacbd3b3c557c00b6c45b710eeddce517a67178b6ba2b", + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + } + ], + "logs": [ + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-pr4_xwks/server-fm85jn50.log", + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-pr4_xwks/server-pjupl8oc.log" + ], + "speed_qualified": false, + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6 + }, + { + "model": "LFM3-1.7B-MLX-6bit", + "binary_sha256": "d1656249bdb581b147faacbd3b3c557c00b6c45b710eeddce517a67178b6ba2b", + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + } + ], + "logs": [ + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-ezqyt_1w/server-80jflpwr.log", + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-ezqyt_1w/server-p_468q3z.log" + ], + "speed_qualified": false, + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6 + }, + { + "model": "LFM3-1.7B-MLX-8bit", + "binary_sha256": "d1656249bdb581b147faacbd3b3c557c00b6c45b710eeddce517a67178b6ba2b", + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + } + ], + "logs": [ + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-199bl30a/server-291bfqu5.log", + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-199bl30a/server-r71mgs16.log" + ], + "speed_qualified": false, + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6 + }, + { + "model": "LFM3-VL-2B-MLX-4bit", + "binary_sha256": "d1656249bdb581b147faacbd3b3c557c00b6c45b710eeddce517a67178b6ba2b", + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + }, + { + "name": "blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef" + } + }, + { + "name": "red_blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40" + } + }, + { + "name": "blue_red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "ab41c21abe0b", + "token_sha256": "ab41c21abe0bf2e4979170d6433bf6e2676eb81c1673f9c96a96a556d8545520", + "output_sha256": "5fd2f9777cad888e3c4ece35ca38402dc33b86a94bf4e7dea9b0ef49a123365c" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + }, + { + "name": "red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + }, + { + "name": "blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef" + } + }, + { + "name": "blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef" + } + }, + { + "name": "red_blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40" + } + }, + { + "name": "red_blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40" + } + }, + { + "name": "blue_red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf" + } + }, + { + "name": "blue_red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9bc07ad8197c", + "token_sha256": "9bc07ad8197cb9d4a6420567b6576b4aeaaf05e8ff576fab3a708c4826d3184f", + "output_sha256": "fb4d9a3ea9e1362803bdb7189900b7e50b593f5df47041771a822f9137b89370" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red" + ], + "observed": [ + "red", + "red", + "red" + ] + } + }, + { + "name": "blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue" + ], + "observed": [ + "blue", + "blue", + "blue" + ] + } + }, + { + "name": "red_blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red", + "blue" + ], + "observed": [ + "red", + "blue" + ] + } + }, + { + "name": "blue_red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue", + "red" + ], + "observed": [ + "blue", + "red" + ] + } + }, + { + "name": "images/concurrent_equals_solos", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + }, + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bcec37a41abb", + "token_sha256": "bcec37a41abba395144f48096269c577079973a1928356de2fd42f3dd6d02408", + "output_sha256": "119d8bb715f7c83eed561340e759f37316cbff470c8aa826b058b4d6bd9b28ef" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "f7ddc7461925", + "token_sha256": "f7ddc7461925c8442c245e0a38577f66aa708992a852b9b81f819caf0304b5b2", + "output_sha256": "76fe0e2af8a801091c6b60b4c11a94c647e639fc3f64e045991beb299ea2bd40" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "436350619b55", + "token_sha256": "436350619b55b33ef20f6a7ce774723e3aaa718328dbdea67982f2245de6f349", + "output_sha256": "a12ce2084797b364b555240ca6ee7053ec7a653d67ee60107ff5fbaffb690caf" + } + ] + }, + { + "name": "images/corrupt_png_rejected", + "status": "passed", + "evidence": { + "status": 400, + "complete": false + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4c9e7e4a0163", + "token_sha256": "4c9e7e4a01637b5054a8cb88295edf6a7697ec454a29a1d4b1e063e85ea643ee", + "output_sha256": "eca8c5913ce3f457045e92b8b046177804d82af0e5833f6b65dda607e3da2095" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d420544d7d90", + "token_sha256": "d420544d7d906a2a1698a1f0df0600d5bed16adb41f30771e510378fca0609ca", + "output_sha256": "ac270e5fb966fbfb96ba953f0c88497ded174bae52f5457b208e5455c56f6d6d" + } + }, + { + "name": "red/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 2, + "server_disconnections_after": 3, + "server_confirmed_cancel": true + } + }, + { + "name": "red/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + }, + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6851da15a29c", + "token_sha256": "6851da15a29cf4d33e588fa208228cd0cf614245e31195187a67dc0b85d68387", + "output_sha256": "b1dc85d579d8dddb7cef7fb30ac8f367d02cfd6fbcbaed9e48573677fee77ead" + } + } + ], + "logs": [ + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-5y8kueq4/server-3jin26ub.log", + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-5y8kueq4/server-gw62zcy4.log" + ], + "speed_qualified": false, + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9 + }, + { + "model": "LFM3-VL-2B-MLX-6bit", + "binary_sha256": "d1656249bdb581b147faacbd3b3c557c00b6c45b710eeddce517a67178b6ba2b", + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + }, + { + "name": "red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + }, + { + "name": "blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0" + } + }, + { + "name": "red_blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5" + } + }, + { + "name": "blue_red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "8b53481ee7d4", + "token_sha256": "8b53481ee7d405c1a3df2b1e75889f087990d6bf2ab9fc7711b12f57cd89d63b", + "output_sha256": "20cbbb99875361fe61bd0834fef3a2e6145115242d0943e6645ada9310f44942" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + }, + { + "name": "red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + }, + { + "name": "red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + }, + { + "name": "blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0" + } + }, + { + "name": "blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0" + } + }, + { + "name": "red_blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5" + } + }, + { + "name": "red_blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5" + } + }, + { + "name": "blue_red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603" + } + }, + { + "name": "blue_red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "d7c4ef4c23b1", + "token_sha256": "d7c4ef4c23b14a34a8ee8caba156ed22dc4842aeefd138131f84091ef3f99deb", + "output_sha256": "f7f24024489e7a275bf5fabecc41af629a76f055ad3434886b70ed2f3ba377d8" + } + ] + }, + { + "name": "red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red" + ], + "observed": [ + "red", + "red" + ] + } + }, + { + "name": "blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue" + ], + "observed": [ + "blue", + "blue" + ] + } + }, + { + "name": "red_blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red", + "blue" + ], + "observed": [ + "red", + "blue" + ] + } + }, + { + "name": "blue_red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue", + "red" + ], + "observed": [ + "blue", + "red", + "blue" + ] + } + }, + { + "name": "images/concurrent_equals_solos", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + }, + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "1ebda8dbe53a", + "token_sha256": "1ebda8dbe53a1f60e352998add4472e43cbd50d1e9827f5e04c4b8d94cddbe1a", + "output_sha256": "3250f8bff0dd329ca4f245e88f024b27b9562e9106d0bae0ec5fe39cb2342bb0" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "4d716fe0a620", + "token_sha256": "4d716fe0a6208d813d10547bb2b0fe6f67936375629c437b71d438815db2aa54", + "output_sha256": "edae4971bed5e05dc1c4d3bc8812b0a54c6285e75a19e9840015b6ddf61aaff5" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "73ebfb49f03d", + "token_sha256": "73ebfb49f03d91c3eceac28dcf4f8e7ae26b09bee8fd7e686b8588e00f8ad421", + "output_sha256": "c4c679768ae53538b9b57e7aedd1f67f4eefd0121cf8bb5ff1557525f3797603" + } + ] + }, + { + "name": "images/corrupt_png_rejected", + "status": "passed", + "evidence": { + "status": 400, + "complete": false + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "red/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 2, + "server_disconnections_after": 3, + "server_confirmed_cancel": true + } + }, + { + "name": "red/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + }, + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c5496640ef02", + "token_sha256": "c5496640ef023df732de814235bdd625043a8a04430bfedecc9d4cab124cde73", + "output_sha256": "f24021cc28a529f519ccb366576b4be11645c4ad7d1f406aab9be260c7e2060f" + } + } + ], + "logs": [ + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-syb56ki9/server-zxfmaxba.log", + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-syb56ki9/server-t2d1ywrf.log" + ], + "speed_qualified": false, + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9 + }, + { + "model": "LFM3-VL-2B-MLX-8bit", + "binary_sha256": "d1656249bdb581b147faacbd3b3c557c00b6c45b710eeddce517a67178b6ba2b", + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + }, + { + "name": "blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6" + } + }, + { + "name": "red_blue/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f" + } + }, + { + "name": "blue_red/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "01966fd5bc9c", + "token_sha256": "01966fd5bc9cbbd3172d981f7131026072addb0c4996d1e2e838afff1d1628da", + "output_sha256": "b269555f9e101f666dec38d5cbe6628bfe1f9c23ca41f635721ab6afc83444bb" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "077371ff4132", + "token_sha256": "077371ff4132fe3eeb4ad7d35eb3d3c06cf4daf570681ee2b68d5058c88671b7", + "output_sha256": "2e28d98dcdf15a2dc9b9e0849e4325b58d672b44cdc9f616b854e767d7c58e5a" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + }, + { + "name": "red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + }, + { + "name": "red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + }, + { + "name": "blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6" + } + }, + { + "name": "blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6" + } + }, + { + "name": "red_blue/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f" + } + }, + { + "name": "red_blue/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f" + } + }, + { + "name": "blue_red/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005" + } + }, + { + "name": "blue_red/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "889c61b36625", + "token_sha256": "889c61b36625fcb8c53108a0d46838bc8cec22817e1b6729ac12269a4fae3bdf", + "output_sha256": "8050dec47bd7aca54d3bf9fdeb552a9ea2c43dd09c39746884f3aa2b17f44eaa" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6272ad3a19e7", + "token_sha256": "6272ad3a19e75ac877cbcd625f56eefac0e420655cc1faa3271e6232b4cab589", + "output_sha256": "14e49326eeb542e28aabc8b91360dd6bcd54def77e449bb02d4c265ee5dacfbe" + } + ] + }, + { + "name": "red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red" + ], + "observed": [ + "red", + "red" + ] + } + }, + { + "name": "blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue" + ], + "observed": [ + "blue", + "blue", + "blue" + ] + } + }, + { + "name": "red_blue/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "red", + "blue" + ], + "observed": [ + "red", + "blue" + ] + } + }, + { + "name": "blue_red/color_semantics", + "status": "passed", + "evidence": { + "expected": [ + "blue", + "red" + ], + "observed": [ + "blue", + "red" + ] + } + }, + { + "name": "images/concurrent_equals_solos", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + }, + { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "443110d479d4", + "token_sha256": "443110d479d465bdf94e0f0593e04fccc97bf6c691a303b18790653d09a2a484", + "output_sha256": "6289ced39f197fde2b527b584d41664646ae63a2b89a7264ef87186c27e7d2d6" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "58a1049f0edd", + "token_sha256": "58a1049f0edda40c71e9d2f870e53b73ae4a684f6e28a01a9c9613cbbb6e87de", + "output_sha256": "c0e8cbed442b0a75be06ef30ae7095b02049d7ea14246713c7ca470ac4a2493f" + }, + { + "prompt_tokens": 166, + "cached_tokens": 128, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "04bedc0c3730", + "token_sha256": "04bedc0c3730c152af9ba4e7b11d4ca29ac0c152e61cc4a537dfc0164418caed", + "output_sha256": "ba7b61007919184772a256b54f96d2d18ffe25be19c24ffd36ecaba9a8a70005" + } + ] + }, + { + "name": "images/corrupt_png_rejected", + "status": "passed", + "evidence": { + "status": 400, + "complete": false + } + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b8b714c6eafc", + "token_sha256": "b8b714c6eafc118c48d77868ccb78c889405ce0d12388774695f4c856da9aed3", + "output_sha256": "6fd3442d9bccd71bdbbdc1882163a95325e8d71deb681db61a61782eafc71258" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "146558fe1b2c", + "token_sha256": "146558fe1b2c249165dcf74e148e01adde16df3443e86827466ab5219ceeda3f", + "output_sha256": "30174e992e3bff60e0fa7976ad3305512b10594afd47a55512eb1b45c95e7955" + } + }, + { + "name": "red/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 2, + "server_disconnections_after": 3, + "server_confirmed_cancel": true + } + }, + { + "name": "red/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + }, + { + "name": "red/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 93, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "5d2a925a67b8", + "token_sha256": "5d2a925a67b8428145573b879dc055195c2182225be42f16e7d54e58ab4eea99", + "output_sha256": "f522a5dd4dea3b4715c1c3c718cf98b96206aa27b286908b94f3c55f63a43d8d" + } + } + ], + "logs": [ + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-04es94s1/server-t6unrjja.log", + "/var/folders/kf/m_08z5_j6mgdn99jsdhtdpww0000gn/T/lfm3-serve-check-04es94s1/server-nniov3zk.log" + ], + "speed_qualified": false, + "counts": { + "passed": 61, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 9 + } + ], + "lifecycle": "One owned model process at a time; every test server exited after its suite." + }, + "limits": [ + "No new tolerance or reduced-precision probability storage.", + "This receipt does not establish a whole-model FP64 fidelity gate or a performance gate.", + "Named sessions and private machine addresses are omitted." + ] +} diff --git a/docs/receipts/lfm3/shared-convolution-text.json b/docs/receipts/lfm3/shared-convolution-text.json new file mode 100644 index 0000000000..e25530e883 --- /dev/null +++ b/docs/receipts/lfm3/shared-convolution-text.json @@ -0,0 +1,43 @@ +{ + "commit": "65051196df3f5d996aae9b4be132240969703dfe", + "scope": "Text-only shared-convolution cherry-pick, actual M4 validation only; no M5 claim", + "sources": { + "zig/kernels/metal/lfm3/ops.metal": "02746819be142ae0788823ffb6963336adc9344fe948d7a9dc94406373e61074", + "zig/src/families/lfm3/forward.zig": "082577207be3a85b3e54011eb7dd2d218e96c28a2f66e969431e4bf3a3f7f8e9", + "zig/src/families/lfm3/kernels.zig": "ad21db3cb7ba951df07b066f6ec244b61a77edb9c8ba8419cf7c786c75de4071", + "zig/src/families/lfm3/state.zig": "13950e1bb0313ae788c43432113d31f27b2b4fa97712d404a3f23ffe2d01ce80", + "zig/tests/contract/metal_main.zig": "e38cedc1d7c8fe134cf14bf422920da0f4612981afda177cc85feedfee4ad2c4", + "zig/tests/contract/metal_lfm3_conv_segments.zig": "b824c468f403e8384942829ace1b6a7c589a4b091a8f2a8f8af6b81244751857", + "zig/tests/contract/metal_limits.zig": "6863bb52b3c48ecd02c0e72a611f9094b94b7e1af9b1642d6c68904ecccb8216", + "zig/src/core/contract/lfm3_records.zig": "ce6f0e8cf8ac7d08e27e825ce9e41d0a0c694875811ef11de5f8f876eaba4c0d" + }, + "build_status": "67/67 steps succeeded; 1/1 newly executed host test passed, others cached", + "build_command": "zig build native tf-kernel-contract test-lfm3-host -Dcpu=apple_m1 --summary all", + "m4_gates": { + "passed": 1, + "failed": 0, + "wrong_state_control_red": 1, + "controls_missed": 0 + }, + "m4_normal_limits": { + "registered_launches": 335, + "problems": 0, + "new_pipeline_max_threads": 256, + "new_pipeline_static_tg_bytes": 0 + }, + "m5_executed": false, + "tree_clean": true, + "binary_sha256": { + "zig-out/native/bin/tensorfold-native": "56b9c8a43342f0d93a73ccefe1e0c8813a9bf19bbe2ef9765380e1ee79c0c21e", + "zig-out/bin/tf-kernel-contract": "86546abb63ddc5ac0de44192db1dff0fd005d3ac9857929075c3519f10400ee5" + }, + "log_sha256": { + "lfm3-shared-conv-text-build.log": "07798b1176d5f6ea8cf3048ef755e55f6b222eb2f33ad42820a0857f5930994f", + "lfm3-shared-conv-text-gates-m4.log": "9b6909521ec6b8665f436367424100eb196a715c4fa3cf1f57353c10d60b0c7e", + "lfm3-shared-conv-text-limits-m4.log": "a7d20c8682be9b90a31adc564ade14832f146a41d43061873d702acbfb59dd27" + }, + "commands": [ + "zig-out/bin/tf-kernel-contract --only \"shared conv\" --no-limits", + "zig-out/bin/tf-kernel-contract --only \"shared conv\" --no-gates" + ] +} diff --git a/docs/receipts/lfm3/shared-convolution.json b/docs/receipts/lfm3/shared-convolution.json new file mode 100644 index 0000000000..fb5d3055c0 --- /dev/null +++ b/docs/receipts/lfm3/shared-convolution.json @@ -0,0 +1,447 @@ +{ + "scope": "Shared-convolution evidence with private qualification/performance and applied-public validation identified separately", + "base_commit": "d05f599a3f341b9bb9bcfbebb9b47b6d63e618ca", + "algorithm": "One descriptor-driven convolution launch across recorded stream segments; both entries invoke the unchanged seven-tap helper and snapshot equations", + "hosts": [ + { + "chip": "Apple M4 Pro", + "macos": "26.5.1", + "build": "25F80", + "memory_gib": 48 + }, + { + "chip": "Apple M5 Ultra", + "macos": "27.0", + "build": "26A428", + "memory_gib": 256 + } + ], + "layout": { + "descriptor_bytes": 16, + "descriptor_alignment": 8, + "offsets": { + "state": 0, + "base": 8, + "rows": 12 + }, + "max_rows": 32, + "scratch_payload_bytes": 12416, + "lifetime": "Separate per-layer descriptor slices and independent per-call row map remain immutable through command completion", + "resources": "Every indirect per-cache convolution-state buffer declared read; input/output/snapshot row spans remain cache-local", + "admission": "Backend init qualification exercises multiple streams and allocates the scratch before native metal_admit.room samples device allocation/footprint; Storage owns and tracks both buffers" + }, + "guards": [ + "Compile-time layout assertions", + "record=false multi-segment calls rejected before encoding, with unchanged cache/history/logits/inputIDs witness on allsix widths", + "Descriptor count and row sums bounded<=32", + "Shared convolution explicitly requires recorded total<=32" + ], + "frozen_private_artifacts": { + "base_commit": "d05f599a3f341b9bb9bcfbebb9b47b6d63e618ca", + "scope": "Private shared-convolution candidate only; defaultOFF unless explicit private environment/testflag", + "source_sha256": { + "zig/kernels/metal/lfm3/ops.metal": "02746819be142ae0788823ffb6963336adc9344fe948d7a9dc94406373e61074", + "zig/src/families/lfm3/model.zig": "7d26a3afe310208ec3b45378890836dac845d262c9186d394951a42906504d60", + "zig/src/families/lfm3/state.zig": "13950e1bb0313ae788c43432113d31f27b2b4fa97712d404a3f23ffe2d01ce80", + "zig/src/families/lfm3/kernels.zig": "4fc6f329a322b8025351b7f097a8dff6757bc6fc4f31b2db32be5960685ce853", + "zig/src/families/lfm3/forward.zig": "e0d2c61773b1e7b6ce5c7ae8d622f65664a3590cddeb750edd99c3f70cc3d32e", + "zig/tests/lfm3_exact.zig": "e3b8902be3b375d8103e9f799051c680219567e871d17c4565bb5a7fb78e2f63" + }, + "binary_sha256": { + "zig-out/bin/tf-lfm3-exact": "98632db29ed6f16eb8b64da8db8a077f8bb7be73beaa7150988a940288844515", + "zig-out/bin/tf-lfm3-forward": "4c0ad68dc32dfa367beb5755be6b1166a23dfc62f0a2203813d6cbdf34e4a548", + "zig-out/native/bin/tensorfold-native": "b06de5679c58f63cffca95c34d0d78ee53678f60e04dff894aded547213ec1a5" + }, + "scratch_bytes": 12416, + "guards": [ + "ConvSegment size16/alignment8/offsets0,8,12 asserted at compile time", + "Unrecorded multi-segment calls rejected before encoding", + "Descriptor row sums validated<=32 before allocation/population", + "Shared launch explicitly requires recorded rows<=32" + ] + }, + "contract_artifact": { + "binary_sha256": "408887b56dc2f899a64f2101341637f01ea0bb8ffdd74a0c3adbc29b664754c7", + "source_sha256": { + "zig/tests/contract/metal_lfm3_conv_segments.zig": "0b1361bfeb044d2033e695d1d5eac3fa2488140b5677ef7a63b8cb172f27f6b2", + "zig/tests/contract/metal_main.zig": "d4b6d164fe1f8c61d5e55c63e8d74ab8a77021b41d5fc2c21c1fea32e0741aab", + "zig/tests/contract/metal_limits.zig": "51c3294d185d69bf56f2d20cadfff50c891bcc2ee4df40e22b2f2e7895694c56", + "zig/src/core/contract/lfm3_records.zig": "a5fa76f575767256030e034a74bc6582d9052180ed6be838bb5736bdc808913b" + }, + "build_steps": [ + 43, + 43 + ] + }, + "serving_scope": "Allsix widths; seeded sampling, actual HTTP cancellation/recovery, unequal/shared streams, prompt-cache reuse and VL image/order cases", + "performance": { + "scope": "Four-stream steady greedy decode on4bit text/VL, two public code/prose chat prompts,256tokens,3reps each leg, ABBA; no speed claim for6/8bit or cold-prefill", + "baseline_sha256": "e9a1488abb130af39c21ee08068fe490d5926271cebcd2e9d0f815711e3730b4", + "candidate_sha256": "b06de5679c58f63cffca95c34d0d78ee53678f60e04dff894aded547213ec1a5", + "order": [ + "old", + "new", + "new", + "old" + ], + "server_options": { + "context": 70000, + "parallel": 4, + "prompt_cache_gib": 0, + "temperature": 0, + "thinking": false + }, + "private_enablement": "TF_LFM3_SHARED_CONV=1 on candidate servers only; no quad/NAX/publication candidate", + "measured_requests": 192, + "counts": { + "alone_equal": 192, + "alone_unequal": 0, + "alone_failed": 0, + "serial_equal": 16, + "serial_unequal": 0, + "serial_failed": 0 + }, + "all_measured_full_token_hashes_match_by_family_prompt": true, + "cells": [ + { + "family": "text", + "prompt": "long-code-chat", + "baseline_leg_rates": [ + 848.2, + 849.5 + ], + "candidate_leg_rates": [ + 877.4, + 876.7 + ], + "baseline_median_tps": 848.85, + "candidate_median_tps": 877.05, + "rate_ratio": 1.0332214172115215, + "token_sha256": "7ffd540d4c865156a86d2dc134d3ac3b2a5c6508f5f7821217f5abcbc89cb9b7" + }, + { + "family": "text", + "prompt": "long-prose-chat", + "baseline_leg_rates": [ + 868.6, + 869.4 + ], + "candidate_leg_rates": [ + 896.7, + 896.2 + ], + "baseline_median_tps": 869.0, + "candidate_median_tps": 896.45, + "rate_ratio": 1.0315880322209436, + "token_sha256": "97f95aa70ea3cdfeb18f05c2b71620d477f3cb71f84ce72236eb1ee8bc636b1a" + }, + { + "family": "vl", + "prompt": "long-code-chat", + "baseline_leg_rates": [ + 825.2, + 824.9 + ], + "candidate_leg_rates": [ + 851.3, + 847.1 + ], + "baseline_median_tps": 825.05, + "candidate_median_tps": 849.2, + "rate_ratio": 1.029270953275559, + "token_sha256": "cd53eb1c94a6051695a27472ab8e927126dacf6884d506996a6ce19c35abe9ee" + }, + { + "family": "vl", + "prompt": "long-prose-chat", + "baseline_leg_rates": [ + 822.2, + 821.1 + ], + "candidate_leg_rates": [ + 846.8, + 843.7 + ], + "baseline_median_tps": 821.6500000000001, + "candidate_median_tps": 845.25, + "rate_ratio": 1.0287226921438568, + "token_sha256": "a6dac318d3938bbe33f79d9a9c09bc030f9da35dc346d8c2d535909b0c3513aa" + } + ], + "runner_sha256": "a50e6058f59fdd5cbca1e937b8bee3c719ae4c201336832425d1aa7787770a64", + "artifact_scope": "Measured on the frozen private opt-in binary; the applied public binary has a distinct SHA and was not this benchmark artifact" + }, + "evidence_sha256": { + "lfm3-shared-conv-contract-m4.log": "d747ee0a98514126816594491d6c2262be22c35c73db61390cb6dda9f38f1e4f", + "lfm3-shared-conv-contract-m5.log": "71c7c162fc0dfd982046f617529a119cd6f568b206110281c126a91e7bffd684", + "lfm3-shared-conv-guard-quick-m4.log": "6b305e0707b3d3867faf380c82f6d5bdbd60326cac431b4813b90ac3308a32fa", + "lfm3-shared-conv-guard-build.log": "bba528bbaa79057884e661f5b832bada7b107dcfdc69c2fd4d19755c6eb666de", + "lfm3-shared-conv-contract-final-build.log": "4eb06fdeef03301fed52b14d74b32b887b6c7fa888f640b6af7f5bb74ebeca0f" + }, + "private_state_suites": [ + { + "model": "LFM3-1.7B-MLX-4bit", + "passed": 68, + "failed": 0, + "skipped": 0, + "unrecorded_rejected_unchanged": true, + "seconds": 69.34337883302942, + "log_sha256": "89f8b71a016ce3eaa2996bc73c6d1dea3f7cb90007f09109b82d346ccdec478e" + }, + { + "model": "LFM3-1.7B-MLX-6bit", + "passed": 68, + "failed": 0, + "skipped": 0, + "unrecorded_rejected_unchanged": true, + "seconds": 81.18592800013721, + "log_sha256": "f0ee8d743c8c22a3e34801e70d0ed2e6e88e23f87f5f7b2380cee48e06b529f8" + }, + { + "model": "LFM3-1.7B-MLX-8bit", + "passed": 68, + "failed": 0, + "skipped": 0, + "unrecorded_rejected_unchanged": true, + "seconds": 83.94439495913684, + "log_sha256": "394710a17d58e50092dcf3d7773946e90a0e5a4018c54da541c0384d67158060" + }, + { + "model": "LFM3-VL-2B-MLX-4bit", + "passed": 68, + "failed": 0, + "skipped": 0, + "unrecorded_rejected_unchanged": true, + "seconds": 73.88264883402735, + "log_sha256": "4b9be778ab98d731a9c0882355ddf670d94e5345e8777121dbf8858fc0a32c79" + }, + { + "model": "LFM3-VL-2B-MLX-6bit", + "passed": 68, + "failed": 0, + "skipped": 0, + "unrecorded_rejected_unchanged": true, + "seconds": 81.54222154105082, + "log_sha256": "69c0c38931a19db1617229aff21d7b352038593f3e364f7e936c980c32d44023" + }, + { + "model": "LFM3-VL-2B-MLX-8bit", + "passed": 68, + "failed": 0, + "skipped": 0, + "unrecorded_rejected_unchanged": true, + "seconds": 83.95681104203686, + "log_sha256": "02d16d94104abfa400ca98f3c6e1ec2edcb1937dd3699ac016b0b80fb87edf1c" + } + ], + "private_state_total": 408, + "private_serving_suites": [ + { + "model": "LFM3-1.7B-MLX-4bit", + "passed": 40, + "failed": 0, + "skipped": 0, + "receipt_sha256": "85e7081cc498d19626f1dbcc0f16a310455d2762f8b202da2f0c5a3606538282" + }, + { + "model": "LFM3-1.7B-MLX-6bit", + "passed": 40, + "failed": 0, + "skipped": 0, + "receipt_sha256": "74f26e831a7b76582f7c8b6ca288f2f31b2bfad4c9836f249a4802af2850920b" + }, + { + "model": "LFM3-1.7B-MLX-8bit", + "passed": 40, + "failed": 0, + "skipped": 0, + "receipt_sha256": "f0f90275348eed175783b3cdcdefb830da3de7984c5751acd0b748b38f2bf92b" + }, + { + "model": "LFM3-VL-2B-MLX-4bit", + "passed": 61, + "failed": 0, + "skipped": 0, + "receipt_sha256": "70115d39e759a7a169c3f300cded1790f445301624fa7f1cbf14d26b5d102075" + }, + { + "model": "LFM3-VL-2B-MLX-6bit", + "passed": 61, + "failed": 0, + "skipped": 0, + "receipt_sha256": "08f4af3efa91ce27b652e3c407bb625cd627af05eda9b85e5b1d80bc74da1ac1" + }, + { + "model": "LFM3-VL-2B-MLX-8bit", + "passed": 61, + "failed": 0, + "skipped": 0, + "receipt_sha256": "582ed6d544d14e058781f1cfa9ce64db36dd3b28520e488583d12b2ff62de8ec" + } + ], + "private_serving_total": 303, + "private_operator_gates": { + "M4": { + "kernels_passed": 1, + "failed": 0, + "controls_red": 1, + "controls_missed": 0, + "nonfinite_classes": 16, + "finite_classes": 114672, + "replacement_shapes": 5, + "repeat_runs": 5, + "repeat_bytes": 229376, + "reduced_precision_witness": "fp16 infx; bf16 963x relative miss", + "limits": { + "launched_threads": 256, + "max_threads": 256, + "static_threadgroup_bytes": 0 + } + }, + "M5": { + "kernels_passed": 1, + "failed": 0, + "controls_red": 1, + "controls_missed": 0, + "limits": { + "launched_threads": 256, + "max_threads": 256, + "static_threadgroup_bytes": 0 + } + }, + "bound": "Existing scalar conv FP64 equation, depth and intermediate-round allowances reused unchanged per independent cache", + "control": "Wrong stream-state address gives raw replacement mismatch and is red as planted", + "limits_scope": "Targeted runs used --no-limits; each new pipeline was explicitly reflected/checked at probe init. Normal gate5 declarations and bad-source compilation are registered for the final full limits run." + }, + "private_immutable_solo_captures": [ + { + "model": "LFM3-1.7B-MLX-4bit", + "rows": 257, + "bytes": 65792000, + "canonical_equal": true, + "sha256": "f1dbd42c2556fcb05031a00b209c61fadee45e74ee1ff94628d4802479fcb518" + }, + { + "model": "LFM3-1.7B-MLX-6bit", + "rows": 257, + "bytes": 65792000, + "canonical_equal": true, + "sha256": "08bf833a94fa0403bc65c70677d20372b5ab0616fb0a58d26119c486069ef59c" + }, + { + "model": "LFM3-1.7B-MLX-8bit", + "rows": 257, + "bytes": 65792000, + "canonical_equal": true, + "sha256": "1345bd80acaa16fe08a00a14d839e3e37cf9d0d8cd7e98c01b68cd1b8d2f51d4" + }, + { + "model": "LFM3-VL-2B-MLX-4bit", + "rows": 257, + "bytes": 65792000, + "canonical_equal": true, + "sha256": "a4dde0a00ee9acb294c2b5a791e7e73cd05c5d6904ffe171e439ad1b01e25c5d" + }, + { + "model": "LFM3-VL-2B-MLX-6bit", + "rows": 257, + "bytes": 65792000, + "canonical_equal": true, + "sha256": "93ea83c90c5623f4e17502a526e2358f9e1ba3f2d03999295745257dc95bbca6" + }, + { + "model": "LFM3-VL-2B-MLX-8bit", + "rows": 257, + "bytes": 65792000, + "canonical_equal": true, + "sha256": "5a013f3eed97f143429f09a81a829b07dc32360734cd54855e18e514160e0a48" + } + ], + "private_immutable_solo_bytes": 394752000, + "applied_public": { + "base_commit": "d05f599a3f341b9bb9bcfbebb9b47b6d63e618ca", + "source_sha256": { + "zig/kernels/metal/lfm3/ops.metal": "02746819be142ae0788823ffb6963336adc9344fe948d7a9dc94406373e61074", + "zig/src/families/lfm3/forward.zig": "082577207be3a85b3e54011eb7dd2d218e96c28a2f66e969431e4bf3a3f7f8e9", + "zig/src/families/lfm3/state.zig": "13950e1bb0313ae788c43432113d31f27b2b4fa97712d404a3f23ffe2d01ce80", + "zig/src/families/lfm3/kernels.zig": "ad21db3cb7ba951df07b066f6ec244b61a77edb9c8ba8419cf7c786c75de4071", + "zig/tests/contract/metal_lfm3_conv_segments.zig": "b824c468f403e8384942829ace1b6a7c589a4b091a8f2a8f8af6b81244751857", + "zig/tests/contract/metal_main.zig": "d4b6d164fe1f8c61d5e55c63e8d74ab8a77021b41d5fc2c21c1fea32e0741aab", + "zig/tests/contract/metal_limits.zig": "51c3294d185d69bf56f2d20cadfff50c891bcc2ee4df40e22b2f2e7895694c56", + "zig/src/core/contract/lfm3_records.zig": "a5fa76f575767256030e034a74bc6582d9052180ed6be838bb5736bdc808913b" + }, + "binary_sha256": { + "zig-out/native/bin/tensorfold-native": "195a8aceefd625f76049ed7cb48459b43f6ebbeedde5e82a07b0726f29010a9d", + "zig-out/bin/tf-lfm3-exact": "7c04d0da0bed247f7bbb08077b74aa190b5a07322c8d54dadfb2898ffbb650b1", + "zig-out/bin/tf-kernel-contract": "31557ec36605bcaf5c40f04218f10ea5374d3cdfb87b6ad644c2ae8cf1139437" + }, + "default": "Shared convolution for recorded multi-segment rounds; unchanged single-stream entry; no private getenv/host-fault flags", + "private_shader_byte_identity": true, + "build": "64/64 steps succeeded", + "operator_and_normal_limits": { + "m4": { + "kernel_gates_passed": 1, + "failed": 0, + "wrong_state_controls_red": 1, + "controls_missed": 0, + "launched_threads": 256, + "pipeline_max_threads": 256, + "static_threadgroup_bytes": 0, + "normal_limits_registered_launches": 353, + "normal_limits_problems": 0, + "scope": "All normal launch declarations against M1-M5 and actual device reflection; no offline M1/M2 compiler sweep" + }, + "m5": { + "kernel_gates_passed": 1, + "failed": 0, + "wrong_state_controls_red": 1, + "controls_missed": 0, + "launched_threads": 256, + "pipeline_max_threads": 256, + "static_threadgroup_bytes": 0, + "normal_limits_registered_launches": 353, + "normal_limits_problems": 0, + "scope": "All normal launch declarations against M1-M5 and actual device reflection; no offline M1/M2 compiler sweep" + } + }, + "state_suite": { + "host": "Apple M4 Pro", + "model": "LFM3-1.7B-MLX-4bit", + "passed": 68, + "failed": 0, + "skipped": 0, + "mode": "default, no enablement flags" + }, + "host_validation": "zig build test-lfm3-host test-lfm3-image-host -Dcpu=apple_m1 --summary all: 10/10 steps; 1/1 newly executed test passed, remaining executions cached", + "fmt": "zig fmt --check on all seven changed Zig sources: passed", + "source_maximum_lines": 468, + "lean": { + "baseline_problems": 428, + "head_problems": 428, + "new_problems": 0, + "note": "Nonzero lean-check findings are inherited; lean_new comparison exits zero", + "baseline_log_sha256": "3d3fc6c70d708ea107e4cd1f87f8674de115b367131a267e68f639006cd0165c" + }, + "commands": [ + "zig build native tf-lfm3-exact tf-kernel-contract -Dcpu=apple_m1 --summary all", + "zig-out/bin/tf-kernel-contract --only \"shared conv\" --no-limits", + "zig-out/bin/tf-kernel-contract --only \"shared conv\" --no-gates", + "zig-out/bin/tf-lfm3-exact MODEL_DIR", + "zig build test-lfm3-host test-lfm3-image-host -Dcpu=apple_m1 --summary all", + "python3 tools/zig/lean_check.py", + "python3 tools/zig/lean_new.py PINNED_BASE_LEAN.txt HEAD_LEAN.txt" + ], + "log_sha256": { + "lfm3-shared-conv-public-build.log": "bfbb9951b04b1fd507c424a9e3d4373f5b7c075b49b26d0b186a18895bb6af07", + "lfm3-shared-conv-public-gates-m4.log": "f351162af6678cf688fabd1a53b6d79bb54229e95ee899d422a55da39cdab541", + "lfm3-shared-conv-public-gates-m5.log": "4d07a2f99af41772509ec501d502bf07e350440074dc4ec675cbbda6b3d3f90d", + "lfm3-shared-conv-public-limits-m4.log": "e7f338d0699f9c2be4aa97ef5995b5bb244bf8cd6046ddd3f91652bf555e6718", + "lfm3-shared-conv-public-limits-m5.log": "89d77a1125cfad02ba0ac94a811989c907a352feeab6a7474b9cae838607ef70", + "lfm3-shared-conv-public-state68-m4.log": "47c399fac3257c8c00d62ab84645cfce8ea44a183af88a1a74d3765bf08e78fd", + "lfm3-shared-conv-public-host-tests.log": "b0b75fac398c6200f54aabc87233d3d4cc1da79b271a98f3b15868a7b52902e4", + "lfm3-shared-conv-public-lean-head.txt": "e570a5cbfae6f29b03a2e746fa5f8141b203ee51412e7b76f9333f05cc658d6f", + "lfm3-shared-conv-public-lean-new.txt": "7c56e1d13027048ee9b6d376556965cbfd2aff0b2b2a67f03612c87643893209", + "lfm3-shared-conv-public-fmt.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + } +} diff --git a/docs/receipts/lfm3/six-width-fp64-decode.json b/docs/receipts/lfm3/six-width-fp64-decode.json new file mode 100644 index 0000000000..fc0f75da81 --- /dev/null +++ b/docs/receipts/lfm3/six-width-fp64-decode.json @@ -0,0 +1,769 @@ +{ + "scope": "Actual one-row decode after 63-row teacher-forced history, all six quantizations; reference re-evaluates identical full history in FP64", + "binary_sha256": "e80a2a14c0d8ed20b62088d0d8b87be4beb5fed3eb1a7336b3b00f8ac5f74bee", + "fidelity_pass": null, + "cells": [ + { + "rows": 1, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.01665075641707762, + "maximum_absolute_error": 0.21875, + "differences": [], + "worst_rows": [ + { + "row": 63, + "rms_error": 0.04319844149784029, + "maximum_absolute_error": 0.21875 + } + ], + "worst_columns": [ + { + "column": 48006, + "rms_error": 0.21875, + "maximum_absolute_error": 0.21875 + }, + { + "column": 30122, + "rms_error": 0.21197509765625, + "maximum_absolute_error": 0.21197509765625 + }, + { + "column": 3568, + "rms_error": 0.20703125, + "maximum_absolute_error": 0.20703125 + }, + { + "column": 42890, + "rms_error": 0.203125, + "maximum_absolute_error": 0.203125 + }, + { + "column": 6723, + "rms_error": 0.203125, + "maximum_absolute_error": 0.203125 + }, + { + "column": 3641, + "rms_error": 0.203125, + "maximum_absolute_error": 0.203125 + }, + { + "column": 7068, + "rms_error": 0.203125, + "maximum_absolute_error": 0.203125 + }, + { + "column": 38765, + "rms_error": 0.203125, + "maximum_absolute_error": 0.203125 + }, + { + "column": 30663, + "rms_error": 0.2001953125, + "maximum_absolute_error": 0.2001953125 + }, + { + "column": 116632, + "rms_error": 0.1953125, + "maximum_absolute_error": 0.1953125 + }, + { + "column": 20560, + "rms_error": 0.1953125, + "maximum_absolute_error": 0.1953125 + }, + { + "column": 20326, + "rms_error": 0.1953125, + "maximum_absolute_error": 0.1953125 + }, + { + "column": 12263, + "rms_error": 0.1953125, + "maximum_absolute_error": 0.1953125 + }, + { + "column": 469, + "rms_error": 0.1875, + "maximum_absolute_error": 0.1875 + }, + { + "column": 11460, + "rms_error": 0.1875, + "maximum_absolute_error": 0.1875 + }, + { + "column": 23796, + "rms_error": 0.1875, + "maximum_absolute_error": 0.1875 + }, + { + "column": 31233, + "rms_error": 0.1875, + "maximum_absolute_error": 0.1875 + }, + { + "column": 36584, + "rms_error": 0.1875, + "maximum_absolute_error": 0.1875 + }, + { + "column": 9178, + "rms_error": 0.1875, + "maximum_absolute_error": 0.1875 + }, + { + "column": 93833, + "rms_error": 0.1875, + "maximum_absolute_error": 0.1875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "checkpoint": "LiquidAI/LFM3-1.7B-MLX-4bit", + "phase": "decode", + "prefilled_history": 63, + "current_row": 63, + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "6d6828e56b463b0a09f9f559ec1251a21ff1445d53e67811dda0318185ce39ff", + "reference_full_logits_sha256": "6286092ef4d800dd2fefe310628c1a5270cc489f33c72a373010b02d38164023" + }, + { + "rows": 1, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.021845787266094638, + "maximum_absolute_error": 0.328125, + "differences": [], + "worst_rows": [ + { + "row": 63, + "rms_error": 0.06023315568323548, + "maximum_absolute_error": 0.328125 + } + ], + "worst_columns": [ + { + "column": 20341, + "rms_error": 0.328125, + "maximum_absolute_error": 0.328125 + }, + { + "column": 35931, + "rms_error": 0.3203125, + "maximum_absolute_error": 0.3203125 + }, + { + "column": 43041, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 8553, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 35725, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 6953, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 15136, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 12603, + "rms_error": 0.310546875, + "maximum_absolute_error": 0.310546875 + }, + { + "column": 9323, + "rms_error": 0.3046875, + "maximum_absolute_error": 0.3046875 + }, + { + "column": 7032, + "rms_error": 0.3046875, + "maximum_absolute_error": 0.3046875 + }, + { + "column": 94782, + "rms_error": 0.296875, + "maximum_absolute_error": 0.296875 + }, + { + "column": 23812, + "rms_error": 0.2939453125, + "maximum_absolute_error": 0.2939453125 + }, + { + "column": 36145, + "rms_error": 0.28125, + "maximum_absolute_error": 0.28125 + }, + { + "column": 93950, + "rms_error": 0.28125, + "maximum_absolute_error": 0.28125 + }, + { + "column": 117795, + "rms_error": 0.28125, + "maximum_absolute_error": 0.28125 + }, + { + "column": 52318, + "rms_error": 0.28125, + "maximum_absolute_error": 0.28125 + }, + { + "column": 94667, + "rms_error": 0.28125, + "maximum_absolute_error": 0.28125 + }, + { + "column": 53589, + "rms_error": 0.28125, + "maximum_absolute_error": 0.28125 + }, + { + "column": 58716, + "rms_error": 0.28125, + "maximum_absolute_error": 0.28125 + }, + { + "column": 32140, + "rms_error": 0.28125, + "maximum_absolute_error": 0.28125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "checkpoint": "LiquidAI/LFM3-1.7B-MLX-6bit", + "phase": "decode", + "prefilled_history": 63, + "current_row": 63, + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "e86739c46d55e1f137ac55c9e43119fda97914902334f12df9069907c1a0cab9", + "reference_full_logits_sha256": "badfeed6f4b81e8f448eb0bd1f523d8dc3f8891e56f0067fe170f566e87eb9e5" + }, + { + "rows": 1, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.03398969064458055, + "maximum_absolute_error": 0.35546875, + "differences": [], + "worst_rows": [ + { + "row": 63, + "rms_error": 0.09125710667798623, + "maximum_absolute_error": 0.35546875 + } + ], + "worst_columns": [ + { + "column": 19987, + "rms_error": 0.35546875, + "maximum_absolute_error": 0.35546875 + }, + { + "column": 21929, + "rms_error": 0.34716796875, + "maximum_absolute_error": 0.34716796875 + }, + { + "column": 7600, + "rms_error": 0.34375, + "maximum_absolute_error": 0.34375 + }, + { + "column": 34767, + "rms_error": 0.34375, + "maximum_absolute_error": 0.34375 + }, + { + "column": 54720, + "rms_error": 0.34375, + "maximum_absolute_error": 0.34375 + }, + { + "column": 29199, + "rms_error": 0.34375, + "maximum_absolute_error": 0.34375 + }, + { + "column": 6143, + "rms_error": 0.33984375, + "maximum_absolute_error": 0.33984375 + }, + { + "column": 60109, + "rms_error": 0.3359375, + "maximum_absolute_error": 0.3359375 + }, + { + "column": 29486, + "rms_error": 0.3330078125, + "maximum_absolute_error": 0.3330078125 + }, + { + "column": 33211, + "rms_error": 0.328125, + "maximum_absolute_error": 0.328125 + }, + { + "column": 54050, + "rms_error": 0.328125, + "maximum_absolute_error": 0.328125 + }, + { + "column": 10936, + "rms_error": 0.328125, + "maximum_absolute_error": 0.328125 + }, + { + "column": 14054, + "rms_error": 0.322540283203125, + "maximum_absolute_error": 0.322540283203125 + }, + { + "column": 46076, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 23955, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 1985, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 63853, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 123494, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 49094, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + }, + { + "column": 10534, + "rms_error": 0.3125, + "maximum_absolute_error": 0.3125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "checkpoint": "LiquidAI/LFM3-1.7B-MLX-8bit", + "phase": "decode", + "prefilled_history": 63, + "current_row": 63, + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "03d2ff5c67ec8bb02539d7fe526dd0bc4f0e9833a8d6474cd1174874fe79c905", + "reference_full_logits_sha256": "e890b65e52b7071b80f6052d520bca1b857466a6453a3b1fdc43df33ea6089eb" + }, + { + "rows": 1, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.030637681948585384, + "maximum_absolute_error": 0.193359375, + "differences": [], + "worst_rows": [ + { + "row": 63, + "rms_error": 0.06189983119754214, + "maximum_absolute_error": 0.193359375 + } + ], + "worst_columns": [ + { + "column": 6095, + "rms_error": 0.193359375, + "maximum_absolute_error": 0.193359375 + }, + { + "column": 6049, + "rms_error": 0.1796875, + "maximum_absolute_error": 0.1796875 + }, + { + "column": 44706, + "rms_error": 0.17919921875, + "maximum_absolute_error": 0.17919921875 + }, + { + "column": 65418, + "rms_error": 0.17578125, + "maximum_absolute_error": 0.17578125 + }, + { + "column": 105864, + "rms_error": 0.171875, + "maximum_absolute_error": 0.171875 + }, + { + "column": 24030, + "rms_error": 0.171875, + "maximum_absolute_error": 0.171875 + }, + { + "column": 42266, + "rms_error": 0.171875, + "maximum_absolute_error": 0.171875 + }, + { + "column": 3136, + "rms_error": 0.171875, + "maximum_absolute_error": 0.171875 + }, + { + "column": 47240, + "rms_error": 0.171875, + "maximum_absolute_error": 0.171875 + }, + { + "column": 1718, + "rms_error": 0.171875, + "maximum_absolute_error": 0.171875 + }, + { + "column": 40928, + "rms_error": 0.171875, + "maximum_absolute_error": 0.171875 + }, + { + "column": 61448, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 108148, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 30586, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 53574, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 12960, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 97794, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 33311, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 27756, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 97838, + "rms_error": 0.158203125, + "maximum_absolute_error": 0.158203125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "phase": "decode", + "prefilled_history": 63, + "current_row": 63, + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "2103df493c16eaf3f7e13751dcf6c2f7bdbe9976f12fa84c6e1a7725053c411e", + "reference_full_logits_sha256": "f5c18f0b75b86711e663904ec4011913ee23bf76b565317e8741de74d5b23dd6" + }, + { + "rows": 1, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.013336827988498537, + "maximum_absolute_error": 0.16796875, + "differences": [], + "worst_rows": [ + { + "row": 63, + "rms_error": 0.028049199550743174, + "maximum_absolute_error": 0.16796875 + } + ], + "worst_columns": [ + { + "column": 2819, + "rms_error": 0.16796875, + "maximum_absolute_error": 0.16796875 + }, + { + "column": 28100, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 15484, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 93, + "rms_error": 0.1640625, + "maximum_absolute_error": 0.1640625 + }, + { + "column": 893, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 74, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 96, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 84, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 8260, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 7511, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 98, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 47335, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 79, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 1384, + "rms_error": 0.142578125, + "maximum_absolute_error": 0.142578125 + }, + { + "column": 7577, + "rms_error": 0.140625, + "maximum_absolute_error": 0.140625 + }, + { + "column": 61735, + "rms_error": 0.140625, + "maximum_absolute_error": 0.140625 + }, + { + "column": 1056, + "rms_error": 0.140625, + "maximum_absolute_error": 0.140625 + }, + { + "column": 51446, + "rms_error": 0.140625, + "maximum_absolute_error": 0.140625 + }, + { + "column": 375, + "rms_error": 0.140625, + "maximum_absolute_error": 0.140625 + }, + { + "column": 59800, + "rms_error": 0.140625, + "maximum_absolute_error": 0.140625 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-6bit", + "phase": "decode", + "prefilled_history": 63, + "current_row": 63, + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "1a033068012e72b9b60be106be6eee6058faaece4b0d439a151b683c1b94c1ef", + "reference_full_logits_sha256": "d16f06c7813c231e4f3b7172456959da94d32adf024381666104fa28c0d3a859" + }, + { + "rows": 1, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.013220059903007179, + "maximum_absolute_error": 0.15625, + "differences": [], + "worst_rows": [ + { + "row": 63, + "rms_error": 0.027668003667474416, + "maximum_absolute_error": 0.15625 + } + ], + "worst_columns": [ + { + "column": 984, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 53, + "rms_error": 0.15625, + "maximum_absolute_error": 0.15625 + }, + { + "column": 42847, + "rms_error": 0.1328125, + "maximum_absolute_error": 0.1328125 + }, + { + "column": 5820, + "rms_error": 0.1328125, + "maximum_absolute_error": 0.1328125 + }, + { + "column": 15460, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 51446, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 48949, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 16019, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 71, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 86, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 15532, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 1443, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 3592, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 3698, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 5872, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 503, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 4353, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 27348, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 1224, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + }, + { + "column": 4507, + "rms_error": 0.125, + "maximum_absolute_error": 0.125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "phase": "decode", + "prefilled_history": 63, + "current_row": 63, + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "472b3ad8ec2237267664292acae4be66f78e80271830962d2d4570e088fcec04", + "reference_full_logits_sha256": "c2263cda17da88664fad1fbb499035c33899760ec5cad02a35a4ba1ac1b908c8" + } + ] +} diff --git a/docs/receipts/lfm3/six-width-fp64-prompt.json b/docs/receipts/lfm3/six-width-fp64-prompt.json new file mode 100644 index 0000000000..e9d71c476e --- /dev/null +++ b/docs/receipts/lfm3/six-width-fp64-prompt.json @@ -0,0 +1,1368 @@ +{ + "scope": "All six quantizations, code64 prompt rows, existing post-RoPE native capture against current direct FP64/BF16 oracle; not final tile qualification", + "fidelity_pass": null, + "accepted_threshold": null, + "cells": [ + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.019898213267257773, + "maximum_absolute_error": 0.6875, + "differences": [], + "worst_rows": [ + { + "row": 0, + "rms_error": 0.12301575847128561, + "maximum_absolute_error": 0.4375 + }, + { + "row": 42, + "rms_error": 0.12141064311438426, + "maximum_absolute_error": 0.625 + }, + { + "row": 41, + "rms_error": 0.10940544488449085, + "maximum_absolute_error": 0.578125 + }, + { + "row": 43, + "rms_error": 0.09222703260603499, + "maximum_absolute_error": 0.390625 + }, + { + "row": 28, + "rms_error": 0.0825923186936875, + "maximum_absolute_error": 0.46875 + }, + { + "row": 62, + "rms_error": 0.07925748348396815, + "maximum_absolute_error": 0.6875 + }, + { + "row": 4, + "rms_error": 0.0773653660968508, + "maximum_absolute_error": 0.40625 + }, + { + "row": 40, + "rms_error": 0.0771670764384705, + "maximum_absolute_error": 0.3778076171875 + }, + { + "row": 22, + "rms_error": 0.07286469848686604, + "maximum_absolute_error": 0.4375 + }, + { + "row": 10, + "rms_error": 0.07064260891454813, + "maximum_absolute_error": 0.28125 + }, + { + "row": 44, + "rms_error": 0.06989413816956311, + "maximum_absolute_error": 0.328125 + }, + { + "row": 23, + "rms_error": 0.06668170145228336, + "maximum_absolute_error": 0.34375 + }, + { + "row": 13, + "rms_error": 0.06271886784844971, + "maximum_absolute_error": 0.296875 + }, + { + "row": 30, + "rms_error": 0.06254545915163834, + "maximum_absolute_error": 0.296875 + }, + { + "row": 1, + "rms_error": 0.06233672771987397, + "maximum_absolute_error": 0.28125 + }, + { + "row": 32, + "rms_error": 0.06086335183898396, + "maximum_absolute_error": 0.40625 + }, + { + "row": 37, + "rms_error": 0.05942772461592182, + "maximum_absolute_error": 0.375 + }, + { + "row": 38, + "rms_error": 0.05940583386453879, + "maximum_absolute_error": 0.38134765625 + }, + { + "row": 6, + "rms_error": 0.05880314112372005, + "maximum_absolute_error": 0.25 + }, + { + "row": 47, + "rms_error": 0.0564171984141962, + "maximum_absolute_error": 0.2578125 + } + ], + "worst_columns": [ + { + "column": 5272, + "rms_error": 0.14924881717078994, + "maximum_absolute_error": 0.4375 + }, + { + "column": 20975, + "rms_error": 0.13830767072011435, + "maximum_absolute_error": 0.375 + }, + { + "column": 15960, + "rms_error": 0.13301721391207205, + "maximum_absolute_error": 0.375 + }, + { + "column": 4484, + "rms_error": 0.13052028163702994, + "maximum_absolute_error": 0.40625 + }, + { + "column": 51466, + "rms_error": 0.12749794184669136, + "maximum_absolute_error": 0.375 + }, + { + "column": 1224, + "rms_error": 0.1270584843006156, + "maximum_absolute_error": 0.5 + }, + { + "column": 15136, + "rms_error": 0.12662081140442283, + "maximum_absolute_error": 0.625 + }, + { + "column": 4361, + "rms_error": 0.12336391993992941, + "maximum_absolute_error": 0.375 + }, + { + "column": 893, + "rms_error": 0.12323758317716488, + "maximum_absolute_error": 0.6875 + }, + { + "column": 8328, + "rms_error": 0.12318601764056089, + "maximum_absolute_error": 0.578125 + }, + { + "column": 8260, + "rms_error": 0.12311975651918346, + "maximum_absolute_error": 0.65625 + }, + { + "column": 22155, + "rms_error": 0.12107258273288721, + "maximum_absolute_error": 0.34375 + }, + { + "column": 15582, + "rms_error": 0.1184407464758198, + "maximum_absolute_error": 0.3515625 + }, + { + "column": 4951, + "rms_error": 0.11740358616322723, + "maximum_absolute_error": 0.5 + }, + { + "column": 94813, + "rms_error": 0.11680805446233773, + "maximum_absolute_error": 0.46875 + }, + { + "column": 1396, + "rms_error": 0.11677431639055184, + "maximum_absolute_error": 0.3304443359375 + }, + { + "column": 38765, + "rms_error": 0.11575825461579775, + "maximum_absolute_error": 0.4375 + }, + { + "column": 7511, + "rms_error": 0.11573834151992225, + "maximum_absolute_error": 0.5 + }, + { + "column": 39382, + "rms_error": 0.11526088490486561, + "maximum_absolute_error": 0.46875 + }, + { + "column": 1367, + "rms_error": 0.1150642205596123, + "maximum_absolute_error": 0.328125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "1.7B-4-code-64", + "phase": "prompt", + "checkpoint": "LiquidAI/LFM3-1.7B-MLX-4bit", + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "f15ce195e69a3e054ba60843b37f105739a6e17f7f5facc13a68892ee2bdd32b", + "reference_logits_sha256": "6286092ef4d800dd2fefe310628c1a5270cc489f33c72a373010b02d38164023" + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.026689922049396048, + "maximum_absolute_error": 0.75, + "differences": [], + "worst_rows": [ + { + "row": 10, + "rms_error": 0.17142728645006744, + "maximum_absolute_error": 0.53125 + }, + { + "row": 0, + "rms_error": 0.1422026444976933, + "maximum_absolute_error": 0.75 + }, + { + "row": 42, + "rms_error": 0.14204671379297865, + "maximum_absolute_error": 0.578125 + }, + { + "row": 16, + "rms_error": 0.13838871068371872, + "maximum_absolute_error": 0.4375 + }, + { + "row": 19, + "rms_error": 0.12012588820733641, + "maximum_absolute_error": 0.34375 + }, + { + "row": 15, + "rms_error": 0.11904887812117255, + "maximum_absolute_error": 0.421875 + }, + { + "row": 31, + "rms_error": 0.10889341164425237, + "maximum_absolute_error": 0.390625 + }, + { + "row": 29, + "rms_error": 0.10141750169027551, + "maximum_absolute_error": 0.4375 + }, + { + "row": 43, + "rms_error": 0.10124202752832445, + "maximum_absolute_error": 0.40625 + }, + { + "row": 49, + "rms_error": 0.09557071156115735, + "maximum_absolute_error": 0.4296875 + }, + { + "row": 14, + "rms_error": 0.09401268020082862, + "maximum_absolute_error": 0.5234375 + }, + { + "row": 2, + "rms_error": 0.09175851789989324, + "maximum_absolute_error": 0.3984375 + }, + { + "row": 8, + "rms_error": 0.09048373547123606, + "maximum_absolute_error": 0.34375 + }, + { + "row": 9, + "rms_error": 0.09024813273761305, + "maximum_absolute_error": 0.4921875 + }, + { + "row": 28, + "rms_error": 0.08802512154159117, + "maximum_absolute_error": 0.46875 + }, + { + "row": 33, + "rms_error": 0.08703463336095042, + "maximum_absolute_error": 0.34375 + }, + { + "row": 51, + "rms_error": 0.08688489918955575, + "maximum_absolute_error": 0.375 + }, + { + "row": 22, + "rms_error": 0.08686510053844272, + "maximum_absolute_error": 0.412109375 + }, + { + "row": 3, + "rms_error": 0.08350938284469905, + "maximum_absolute_error": 0.375 + }, + { + "row": 56, + "rms_error": 0.08325712606026651, + "maximum_absolute_error": 0.375 + } + ], + "worst_columns": [ + { + "column": 88140, + "rms_error": 0.21171019029755553, + "maximum_absolute_error": 0.45989990234375 + }, + { + "column": 88304, + "rms_error": 0.20554743382484159, + "maximum_absolute_error": 0.5234375 + }, + { + "column": 115125, + "rms_error": 0.19978058819755923, + "maximum_absolute_error": 0.4375 + }, + { + "column": 49899, + "rms_error": 0.19276769981232406, + "maximum_absolute_error": 0.375 + }, + { + "column": 90359, + "rms_error": 0.19056538429877645, + "maximum_absolute_error": 0.421875 + }, + { + "column": 91957, + "rms_error": 0.18814929803474448, + "maximum_absolute_error": 0.390625 + }, + { + "column": 20975, + "rms_error": 0.18652640335361384, + "maximum_absolute_error": 0.4375 + }, + { + "column": 89889, + "rms_error": 0.18440182377581552, + "maximum_absolute_error": 0.32421875 + }, + { + "column": 22497, + "rms_error": 0.18380758332062028, + "maximum_absolute_error": 0.375 + }, + { + "column": 14310, + "rms_error": 0.18365566751680795, + "maximum_absolute_error": 0.375 + }, + { + "column": 4484, + "rms_error": 0.18060141843418825, + "maximum_absolute_error": 0.34375 + }, + { + "column": 53417, + "rms_error": 0.18054394272788096, + "maximum_absolute_error": 0.4453125 + }, + { + "column": 53842, + "rms_error": 0.1796946342752001, + "maximum_absolute_error": 0.34375 + }, + { + "column": 116115, + "rms_error": 0.17918524830590607, + "maximum_absolute_error": 0.453125 + }, + { + "column": 56370, + "rms_error": 0.1777892499616714, + "maximum_absolute_error": 0.421875 + }, + { + "column": 91849, + "rms_error": 0.17767817334408556, + "maximum_absolute_error": 0.375 + }, + { + "column": 15960, + "rms_error": 0.17582125738297258, + "maximum_absolute_error": 0.375 + }, + { + "column": 87970, + "rms_error": 0.17520242600049366, + "maximum_absolute_error": 0.40625 + }, + { + "column": 117071, + "rms_error": 0.17350036280879913, + "maximum_absolute_error": 0.3515625 + }, + { + "column": 17639, + "rms_error": 0.17306008396153286, + "maximum_absolute_error": 0.375 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "1.7B-6-code-64", + "phase": "prompt", + "checkpoint": "LiquidAI/LFM3-1.7B-MLX-6bit", + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "f94b78c965c874c13fee175b036265f5691b7711b57559f942ecad80e3f1d4a3", + "reference_logits_sha256": "badfeed6f4b81e8f448eb0bd1f523d8dc3f8891e56f0067fe170f566e87eb9e5" + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.0220501312407529, + "maximum_absolute_error": 0.6640625, + "differences": [], + "worst_rows": [ + { + "row": 41, + "rms_error": 0.17345533863740256, + "maximum_absolute_error": 0.59375 + }, + { + "row": 0, + "rms_error": 0.16594398697398952, + "maximum_absolute_error": 0.6640625 + }, + { + "row": 28, + "rms_error": 0.11302524768018539, + "maximum_absolute_error": 0.53125 + }, + { + "row": 22, + "rms_error": 0.0966248427386952, + "maximum_absolute_error": 0.5 + }, + { + "row": 62, + "rms_error": 0.09545741607618395, + "maximum_absolute_error": 0.46875 + }, + { + "row": 63, + "rms_error": 0.09125710667798623, + "maximum_absolute_error": 0.35546875 + }, + { + "row": 15, + "rms_error": 0.09025357831733094, + "maximum_absolute_error": 0.3125 + }, + { + "row": 16, + "rms_error": 0.0888772476755528, + "maximum_absolute_error": 0.3125 + }, + { + "row": 47, + "rms_error": 0.08819708714972854, + "maximum_absolute_error": 0.375 + }, + { + "row": 4, + "rms_error": 0.08449832876546694, + "maximum_absolute_error": 0.53125 + }, + { + "row": 42, + "rms_error": 0.07635969195498178, + "maximum_absolute_error": 0.40625 + }, + { + "row": 32, + "rms_error": 0.0721201318620769, + "maximum_absolute_error": 0.34375 + }, + { + "row": 61, + "rms_error": 0.06953682581797835, + "maximum_absolute_error": 0.34375 + }, + { + "row": 31, + "rms_error": 0.0695276687756236, + "maximum_absolute_error": 0.4375 + }, + { + "row": 33, + "rms_error": 0.06872247151234558, + "maximum_absolute_error": 0.34375 + }, + { + "row": 29, + "rms_error": 0.06841369688082365, + "maximum_absolute_error": 0.38037109375 + }, + { + "row": 43, + "rms_error": 0.06832428159781277, + "maximum_absolute_error": 0.3828125 + }, + { + "row": 21, + "rms_error": 0.06749965844760528, + "maximum_absolute_error": 0.34375 + }, + { + "row": 1, + "rms_error": 0.06727780028766155, + "maximum_absolute_error": 0.28125 + }, + { + "row": 40, + "rms_error": 0.06704998690284182, + "maximum_absolute_error": 0.390625 + } + ], + "worst_columns": [ + { + "column": 42678, + "rms_error": 0.12685415371513498, + "maximum_absolute_error": 0.4296875 + }, + { + "column": 33237, + "rms_error": 0.12621574465901272, + "maximum_absolute_error": 0.5625 + }, + { + "column": 116115, + "rms_error": 0.1260394301754644, + "maximum_absolute_error": 0.3125 + }, + { + "column": 4625, + "rms_error": 0.12559285241247872, + "maximum_absolute_error": 0.375 + }, + { + "column": 20975, + "rms_error": 0.12487812452403631, + "maximum_absolute_error": 0.3125 + }, + { + "column": 61089, + "rms_error": 0.12359072449449683, + "maximum_absolute_error": 0.4375 + }, + { + "column": 46015, + "rms_error": 0.12329067593707133, + "maximum_absolute_error": 0.5 + }, + { + "column": 22155, + "rms_error": 0.12240827791801107, + "maximum_absolute_error": 0.375 + }, + { + "column": 28678, + "rms_error": 0.12169910543090903, + "maximum_absolute_error": 0.484375 + }, + { + "column": 39382, + "rms_error": 0.12169767437400192, + "maximum_absolute_error": 0.59375 + }, + { + "column": 30270, + "rms_error": 0.12051623050711183, + "maximum_absolute_error": 0.5 + }, + { + "column": 116016, + "rms_error": 0.11991592954169991, + "maximum_absolute_error": 0.4375 + }, + { + "column": 91493, + "rms_error": 0.11961819347760586, + "maximum_absolute_error": 0.4375 + }, + { + "column": 92555, + "rms_error": 0.11936493379283061, + "maximum_absolute_error": 0.3125 + }, + { + "column": 9323, + "rms_error": 0.1187778901324646, + "maximum_absolute_error": 0.421875 + }, + { + "column": 10835, + "rms_error": 0.11848167420983581, + "maximum_absolute_error": 0.375 + }, + { + "column": 93981, + "rms_error": 0.11821733014965576, + "maximum_absolute_error": 0.375 + }, + { + "column": 5018, + "rms_error": 0.11756790725059593, + "maximum_absolute_error": 0.5546875 + }, + { + "column": 93924, + "rms_error": 0.11752106955131514, + "maximum_absolute_error": 0.6171875 + }, + { + "column": 119658, + "rms_error": 0.11746209080912955, + "maximum_absolute_error": 0.3046875 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "1.7B-8-code-64", + "phase": "prompt", + "checkpoint": "LiquidAI/LFM3-1.7B-MLX-8bit", + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "c0bc4bfb20a4a24abe52d4b8366c9a4acdabd597b68f627552bc0d9b06c7450b", + "reference_logits_sha256": "e890b65e52b7071b80f6052d520bca1b857466a6453a3b1fdc43df33ea6089eb" + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.013956257000991174, + "maximum_absolute_error": 0.6875, + "differences": [], + "worst_rows": [ + { + "row": 38, + "rms_error": 0.11460334310170639, + "maximum_absolute_error": 0.328125 + }, + { + "row": 29, + "rms_error": 0.07609650779142937, + "maximum_absolute_error": 0.248291015625 + }, + { + "row": 0, + "rms_error": 0.07317533579110798, + "maximum_absolute_error": 0.265625 + }, + { + "row": 57, + "rms_error": 0.06819346713585736, + "maximum_absolute_error": 0.6875 + }, + { + "row": 11, + "rms_error": 0.06719930161309025, + "maximum_absolute_error": 0.296875 + }, + { + "row": 63, + "rms_error": 0.06189983119754214, + "maximum_absolute_error": 0.193359375 + }, + { + "row": 17, + "rms_error": 0.0562844409253447, + "maximum_absolute_error": 0.234375 + }, + { + "row": 55, + "rms_error": 0.055719994631985244, + "maximum_absolute_error": 0.21875 + }, + { + "row": 47, + "rms_error": 0.05486522860095206, + "maximum_absolute_error": 0.21875 + }, + { + "row": 15, + "rms_error": 0.05260483982982732, + "maximum_absolute_error": 0.375 + }, + { + "row": 8, + "rms_error": 0.051856505198323556, + "maximum_absolute_error": 0.25 + }, + { + "row": 21, + "rms_error": 0.05108993972236896, + "maximum_absolute_error": 0.28125 + }, + { + "row": 28, + "rms_error": 0.05107495391341613, + "maximum_absolute_error": 0.46875 + }, + { + "row": 61, + "rms_error": 0.051041773960293906, + "maximum_absolute_error": 0.3125 + }, + { + "row": 12, + "rms_error": 0.050840759407169596, + "maximum_absolute_error": 0.2109375 + }, + { + "row": 35, + "rms_error": 0.050085313313313694, + "maximum_absolute_error": 0.4375 + }, + { + "row": 46, + "rms_error": 0.049754913376188196, + "maximum_absolute_error": 0.375 + }, + { + "row": 39, + "rms_error": 0.04662994047939181, + "maximum_absolute_error": 0.21875 + }, + { + "row": 27, + "rms_error": 0.04651380550464994, + "maximum_absolute_error": 0.19140625 + }, + { + "row": 62, + "rms_error": 0.04637423724016495, + "maximum_absolute_error": 0.21875 + } + ], + "worst_columns": [ + { + "column": 851, + "rms_error": 0.12639365530749455, + "maximum_absolute_error": 0.6875 + }, + { + "column": 33237, + "rms_error": 0.10850725640146129, + "maximum_absolute_error": 0.375 + }, + { + "column": 16033, + "rms_error": 0.10608651587427771, + "maximum_absolute_error": 0.28125 + }, + { + "column": 1011, + "rms_error": 0.10503102966000476, + "maximum_absolute_error": 0.53125 + }, + { + "column": 8373, + "rms_error": 0.10337083627736275, + "maximum_absolute_error": 0.5625 + }, + { + "column": 723, + "rms_error": 0.1002562220633136, + "maximum_absolute_error": 0.4375 + }, + { + "column": 40928, + "rms_error": 0.09699366981465977, + "maximum_absolute_error": 0.5 + }, + { + "column": 1583, + "rms_error": 0.09649414292256268, + "maximum_absolute_error": 0.46875 + }, + { + "column": 295, + "rms_error": 0.09053799629291337, + "maximum_absolute_error": 0.359375 + }, + { + "column": 116016, + "rms_error": 0.09029335147521579, + "maximum_absolute_error": 0.265625 + }, + { + "column": 2690, + "rms_error": 0.08823780691693173, + "maximum_absolute_error": 0.375 + }, + { + "column": 45707, + "rms_error": 0.08812709362843722, + "maximum_absolute_error": 0.265625 + }, + { + "column": 42847, + "rms_error": 0.08731298300569229, + "maximum_absolute_error": 0.3125 + }, + { + "column": 11448, + "rms_error": 0.08659296791732592, + "maximum_absolute_error": 0.25 + }, + { + "column": 21516, + "rms_error": 0.08597371167496128, + "maximum_absolute_error": 0.25 + }, + { + "column": 98225, + "rms_error": 0.08511426945166548, + "maximum_absolute_error": 0.25390625 + }, + { + "column": 11460, + "rms_error": 0.08477921618656097, + "maximum_absolute_error": 0.265625 + }, + { + "column": 17788, + "rms_error": 0.08402464145310369, + "maximum_absolute_error": 0.38671875 + }, + { + "column": 30270, + "rms_error": 0.08306129079244248, + "maximum_absolute_error": 0.28125 + }, + { + "column": 6470, + "rms_error": 0.0826058937474372, + "maximum_absolute_error": 0.4375 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-4-code-64", + "phase": "prompt", + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "14a4249434dcf626b1151fa9d291256ea6dbeb8005c9b2211cff0aefb6e29722", + "reference_logits_sha256": "f5c18f0b75b86711e663904ec4011913ee23bf76b565317e8741de74d5b23dd6" + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 2, + "disjoint_maximum_sets": 1, + "relative_rms": 0.015377440488712887, + "maximum_absolute_error": 0.96484375, + "differences": [ + { + "row": 13, + "native_top": 22, + "reference_top": 63908, + "native_maximum_set": [ + 22, + 63908 + ], + "reference_maximum_set": [ + 63908 + ], + "maximum_sets_overlap": true, + "native_margin": 0.0, + "reference_margin": 0.0625, + "native_reference_contender_gap": 0.0, + "reference_native_contender_gap": 0.0625 + }, + { + "row": 22, + "native_top": 16033, + "reference_top": 91, + "native_maximum_set": [ + 16033 + ], + "reference_maximum_set": [ + 91 + ], + "maximum_sets_overlap": false, + "native_margin": 0.125, + "reference_margin": 0.1875, + "native_reference_contender_gap": 0.125, + "reference_native_contender_gap": 0.1875 + } + ], + "worst_rows": [ + { + "row": 57, + "rms_error": 0.16603965592691475, + "maximum_absolute_error": 0.96484375 + }, + { + "row": 55, + "rms_error": 0.08489186681149496, + "maximum_absolute_error": 0.3046875 + }, + { + "row": 13, + "rms_error": 0.07917376823524733, + "maximum_absolute_error": 0.28125 + }, + { + "row": 2, + "rms_error": 0.06311253390330333, + "maximum_absolute_error": 0.216796875 + }, + { + "row": 32, + "rms_error": 0.060344014781392605, + "maximum_absolute_error": 0.28125 + }, + { + "row": 22, + "rms_error": 0.055970470902545984, + "maximum_absolute_error": 0.375 + }, + { + "row": 43, + "rms_error": 0.053908494189525466, + "maximum_absolute_error": 0.2109375 + }, + { + "row": 53, + "rms_error": 0.05371347456512, + "maximum_absolute_error": 0.2109375 + }, + { + "row": 50, + "rms_error": 0.05305531735124143, + "maximum_absolute_error": 0.1875 + }, + { + "row": 46, + "rms_error": 0.05286167453834549, + "maximum_absolute_error": 0.29296875 + }, + { + "row": 28, + "rms_error": 0.05198421343833805, + "maximum_absolute_error": 0.28125 + }, + { + "row": 56, + "rms_error": 0.050748718923888735, + "maximum_absolute_error": 0.203125 + }, + { + "row": 5, + "rms_error": 0.04965621029864758, + "maximum_absolute_error": 0.1669769287109375 + }, + { + "row": 34, + "rms_error": 0.04927833272243707, + "maximum_absolute_error": 0.21875 + }, + { + "row": 36, + "rms_error": 0.04908556341435462, + "maximum_absolute_error": 0.1875 + }, + { + "row": 4, + "rms_error": 0.0480743780157286, + "maximum_absolute_error": 0.294921875 + }, + { + "row": 62, + "rms_error": 0.04713935283875158, + "maximum_absolute_error": 0.3125 + }, + { + "row": 1, + "rms_error": 0.0456394677394624, + "maximum_absolute_error": 0.1875 + }, + { + "row": 33, + "rms_error": 0.0447680490608537, + "maximum_absolute_error": 0.21875 + }, + { + "row": 38, + "rms_error": 0.044176922297933914, + "maximum_absolute_error": 0.25 + } + ], + "worst_columns": [ + { + "column": 93538, + "rms_error": 0.14026637509168713, + "maximum_absolute_error": 0.96484375 + }, + { + "column": 538, + "rms_error": 0.13238456448014496, + "maximum_absolute_error": 0.953125 + }, + { + "column": 1046, + "rms_error": 0.1223944061953623, + "maximum_absolute_error": 0.5625 + }, + { + "column": 94852, + "rms_error": 0.1214250871538932, + "maximum_absolute_error": 0.828125 + }, + { + "column": 7272, + "rms_error": 0.11949435818918488, + "maximum_absolute_error": 0.84375 + }, + { + "column": 17, + "rms_error": 0.11716969672707923, + "maximum_absolute_error": 0.8125 + }, + { + "column": 6907, + "rms_error": 0.11613526003940478, + "maximum_absolute_error": 0.849609375 + }, + { + "column": 346, + "rms_error": 0.11392491089664815, + "maximum_absolute_error": 0.78125 + }, + { + "column": 16997, + "rms_error": 0.11332787179175711, + "maximum_absolute_error": 0.6875 + }, + { + "column": 3585, + "rms_error": 0.11256296995771509, + "maximum_absolute_error": 0.8125 + }, + { + "column": 647, + "rms_error": 0.11219731048893868, + "maximum_absolute_error": 0.796875 + }, + { + "column": 52421, + "rms_error": 0.10928653461278572, + "maximum_absolute_error": 0.796875 + }, + { + "column": 9673, + "rms_error": 0.10862716394009563, + "maximum_absolute_error": 0.7734375 + }, + { + "column": 13685, + "rms_error": 0.10791157272755565, + "maximum_absolute_error": 0.76953125 + }, + { + "column": 1954, + "rms_error": 0.10729993054205725, + "maximum_absolute_error": 0.767578125 + }, + { + "column": 1062, + "rms_error": 0.10728559453153616, + "maximum_absolute_error": 0.78125 + }, + { + "column": 7415, + "rms_error": 0.10701046101345699, + "maximum_absolute_error": 0.734375 + }, + { + "column": 5728, + "rms_error": 0.1068937588477719, + "maximum_absolute_error": 0.765625 + }, + { + "column": 48564, + "rms_error": 0.10625993545718224, + "maximum_absolute_error": 0.76806640625 + }, + { + "column": 34398, + "rms_error": 0.10505060026867931, + "maximum_absolute_error": 0.7001953125 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-6-code-64", + "phase": "prompt", + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-6bit", + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "02225f1c8c7f5de05ebd44bd49948ad000aac040f3d6025d6655a9772ed08f9b", + "reference_logits_sha256": "d16f06c7813c231e4f3b7172456959da94d32adf024381666104fa28c0d3a859" + }, + { + "rows": 64, + "vocabulary": 128000, + "top1_differences": 0, + "disjoint_maximum_sets": 0, + "relative_rms": 0.015466008538336544, + "maximum_absolute_error": 0.6201171875, + "differences": [], + "worst_rows": [ + { + "row": 57, + "rms_error": 0.10084401675170394, + "maximum_absolute_error": 0.6201171875 + }, + { + "row": 54, + "rms_error": 0.08537710665171272, + "maximum_absolute_error": 0.3359375 + }, + { + "row": 4, + "rms_error": 0.0847654133912914, + "maximum_absolute_error": 0.3828125 + }, + { + "row": 28, + "rms_error": 0.07578738582927384, + "maximum_absolute_error": 0.25 + }, + { + "row": 19, + "rms_error": 0.07325385829626402, + "maximum_absolute_error": 0.322265625 + }, + { + "row": 15, + "rms_error": 0.06881209672444469, + "maximum_absolute_error": 0.2587890625 + }, + { + "row": 60, + "rms_error": 0.06712113892820007, + "maximum_absolute_error": 0.21875 + }, + { + "row": 59, + "rms_error": 0.06689059146927472, + "maximum_absolute_error": 0.25 + }, + { + "row": 0, + "rms_error": 0.06643536714740046, + "maximum_absolute_error": 0.28515625 + }, + { + "row": 33, + "rms_error": 0.0644964015744809, + "maximum_absolute_error": 0.203125 + }, + { + "row": 18, + "rms_error": 0.06176497570554598, + "maximum_absolute_error": 0.2734375 + }, + { + "row": 41, + "rms_error": 0.05864792858652117, + "maximum_absolute_error": 0.25 + }, + { + "row": 42, + "rms_error": 0.05249413812563852, + "maximum_absolute_error": 0.28125 + }, + { + "row": 56, + "rms_error": 0.05004946561577254, + "maximum_absolute_error": 0.2578125 + }, + { + "row": 29, + "rms_error": 0.04989588211022091, + "maximum_absolute_error": 0.25 + }, + { + "row": 55, + "rms_error": 0.04815518751565306, + "maximum_absolute_error": 0.296875 + }, + { + "row": 26, + "rms_error": 0.0477324446813435, + "maximum_absolute_error": 0.232421875 + }, + { + "row": 3, + "rms_error": 0.04683582065091273, + "maximum_absolute_error": 0.2734375 + }, + { + "row": 22, + "rms_error": 0.04634210225231695, + "maximum_absolute_error": 0.3125 + }, + { + "row": 46, + "rms_error": 0.04595433784186023, + "maximum_absolute_error": 0.3125 + } + ], + "worst_columns": [ + { + "column": 295, + "rms_error": 0.1137957235809952, + "maximum_absolute_error": 0.6015625 + }, + { + "column": 16033, + "rms_error": 0.1029568109841047, + "maximum_absolute_error": 0.34375 + }, + { + "column": 833, + "rms_error": 0.10202480514875116, + "maximum_absolute_error": 0.40625 + }, + { + "column": 723, + "rms_error": 0.10091368970147231, + "maximum_absolute_error": 0.5 + }, + { + "column": 42847, + "rms_error": 0.1004209925219702, + "maximum_absolute_error": 0.375 + }, + { + "column": 945, + "rms_error": 0.09906024780067336, + "maximum_absolute_error": 0.53515625 + }, + { + "column": 958, + "rms_error": 0.09743958757285678, + "maximum_absolute_error": 0.6201171875 + }, + { + "column": 60912, + "rms_error": 0.09729450010987249, + "maximum_absolute_error": 0.609375 + }, + { + "column": 5081, + "rms_error": 0.09665967271722023, + "maximum_absolute_error": 0.59375 + }, + { + "column": 538, + "rms_error": 0.09560578125073615, + "maximum_absolute_error": 0.56494140625 + }, + { + "column": 1492, + "rms_error": 0.09468376529995495, + "maximum_absolute_error": 0.46875 + }, + { + "column": 90978, + "rms_error": 0.09393744802568715, + "maximum_absolute_error": 0.34375 + }, + { + "column": 2551, + "rms_error": 0.09376275949274417, + "maximum_absolute_error": 0.59375 + }, + { + "column": 2024, + "rms_error": 0.09340610767263444, + "maximum_absolute_error": 0.53125 + }, + { + "column": 3570, + "rms_error": 0.09310916586806178, + "maximum_absolute_error": 0.5 + }, + { + "column": 33237, + "rms_error": 0.09252003646045862, + "maximum_absolute_error": 0.25 + }, + { + "column": 4673, + "rms_error": 0.0914817433096805, + "maximum_absolute_error": 0.4375 + }, + { + "column": 346, + "rms_error": 0.09074473601256952, + "maximum_absolute_error": 0.4375 + }, + { + "column": 8811, + "rms_error": 0.0905887564170753, + "maximum_absolute_error": 0.5546875 + }, + { + "column": 293, + "rms_error": 0.09005452466329847, + "maximum_absolute_error": 0.3984375 + } + ], + "accepted_threshold": null, + "fidelity_pass": null, + "cell": "VL-2B-8-code-64", + "phase": "prompt", + "checkpoint": "LiquidAI/LFM3-VL-2B-MLX-8bit", + "token_fixture_sha256": "cce7d631dcbf26f43abb6d551a62e51c06582f958a5b5be57b7b5e32e6760748", + "native_logits_sha256": "cb2b3a0590dbcb1ee4ea5914786a9a9954fb06e8ef9f16230cc0f097b5c2bb64", + "reference_logits_sha256": "c2263cda17da88664fad1fbb499035c33899760ec5cad02a35a4ba1ac1b908c8" + } + ], + "native_diagnostic_sha256": "67750e460820140f18649f3a6b93a9f379be152d5cb90d414a97bb339586b655" +} diff --git a/docs/receipts/lfm3/structured-token-checks.json b/docs/receipts/lfm3/structured-token-checks.json new file mode 100644 index 0000000000..a68887914c --- /dev/null +++ b/docs/receipts/lfm3/structured-token-checks.json @@ -0,0 +1,4480 @@ +{ + "cells": [ + { + "platform": "M4 Pro, macOS 26.5.1", + "checkpoint": "1.7B-4", + "native_binary_sha256": "5c8a04d4af599879797f4f5bed8e00ab7d9d70732e698bd2a1ec2a01ec762e09", + "original_client_exit": 0, + "groups": 32, + "passed_full_token_equality": 32, + "failed_full_token_equality": 0, + "checks": [ + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "temperature": 0 + }, + "count": 3, + "hashes": [ + "97fc568e1c13add8c4f1c58c38ab18a87039408de0d789a92614693a8f5c73d0", + "97fc568e1c13add8c4f1c58c38ab18a87039408de0d789a92614693a8f5c73d0", + "97fc568e1c13add8c4f1c58c38ab18a87039408de0d789a92614693a8f5c73d0" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "8f3cef2f2dd40372a9f5a5a45465d005711e83e73de634020964ea1164f102fc", + "8f3cef2f2dd40372a9f5a5a45465d005711e83e73de634020964ea1164f102fc", + "8f3cef2f2dd40372a9f5a5a45465d005711e83e73de634020964ea1164f102fc" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "28e9992d945cd22c289c5a1604e9d84b8fd4d93908d1d3538e441728c2d4b149", + "28e9992d945cd22c289c5a1604e9d84b8fd4d93908d1d3538e441728c2d4b149", + "28e9992d945cd22c289c5a1604e9d84b8fd4d93908d1d3538e441728c2d4b149" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "cda680f6cc3d90448f12337ac2cc1b0119f16da48a71ace0e8e1f192c9945a6d", + "cda680f6cc3d90448f12337ac2cc1b0119f16da48a71ace0e8e1f192c9945a6d", + "cda680f6cc3d90448f12337ac2cc1b0119f16da48a71ace0e8e1f192c9945a6d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "What is the weather like in Lisbon today? Answer as JSON.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "json_schema": { + "name": "weather", + "schema": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + } + }, + "type": "json_schema" + }, + "temperature": 0 + }, + "count": 3, + "hashes": [ + "d455a572b3e0b29505b67549d22ee6dbdaa3453a3f3d4c4a74d5f9c53c288ac7", + "d455a572b3e0b29505b67549d22ee6dbdaa3453a3f3d4c4a74d5f9c53c288ac7", + "d455a572b3e0b29505b67549d22ee6dbdaa3453a3f3d4c4a74d5f9c53c288ac7" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "What is the weather like in Lisbon today? Answer as JSON.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "json_schema": { + "name": "weather", + "schema": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + } + }, + "type": "json_schema" + }, + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "a6703c6b4506a475098af2e37ca37cc66f508ffd46238e5f76546b1ed795021d", + "a6703c6b4506a475098af2e37ca37cc66f508ffd46238e5f76546b1ed795021d", + "a6703c6b4506a475098af2e37ca37cc66f508ffd46238e5f76546b1ed795021d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "What is the weather like in Lisbon today? Answer as JSON.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "json_schema": { + "name": "weather", + "schema": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + } + }, + "type": "json_schema" + }, + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "e2ced68110d55bbdfede82e2431832288dc53c5f2ac44e0f99e4cf28ada34b1c", + "e2ced68110d55bbdfede82e2431832288dc53c5f2ac44e0f99e4cf28ada34b1c", + "e2ced68110d55bbdfede82e2431832288dc53c5f2ac44e0f99e4cf28ada34b1c" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "What is the weather like in Lisbon today? Answer as JSON.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "json_schema": { + "name": "weather", + "schema": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + } + }, + "type": "json_schema" + }, + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "585b2f3c6566d27343b7af1c4464f280c389a54f26ed878de701db5c0705a147", + "585b2f3c6566d27343b7af1c4464f280c389a54f26ed878de701db5c0705a147", + "585b2f3c6566d27343b7af1c4464f280c389a54f26ed878de701db5c0705a147" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_json": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + }, + "max_tokens": 400, + "messages": [ + { + "content": "Make up the weather in Oslo.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "d2bf435d203b8e78856bb7eae3e2e7b7c31ae53d883fcf6564fb58f1a938376c", + "d2bf435d203b8e78856bb7eae3e2e7b7c31ae53d883fcf6564fb58f1a938376c", + "d2bf435d203b8e78856bb7eae3e2e7b7c31ae53d883fcf6564fb58f1a938376c" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_json": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Make up the weather in Oslo.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "2febc228486dea3590e75587cb09af791e2c289b3f1f589ba69b4d81100c390d", + "2febc228486dea3590e75587cb09af791e2c289b3f1f589ba69b4d81100c390d", + "2febc228486dea3590e75587cb09af791e2c289b3f1f589ba69b4d81100c390d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_json": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + }, + "max_tokens": 400, + "messages": [ + { + "content": "Make up the weather in Oslo.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "4faffa1eba4aecc775cd93a9b87f1189e4df90760c98d364a69b9b0b07223162", + "4faffa1eba4aecc775cd93a9b87f1189e4df90760c98d364a69b9b0b07223162", + "4faffa1eba4aecc775cd93a9b87f1189e4df90760c98d364a69b9b0b07223162" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_json": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Make up the weather in Oslo.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "992b757e11570bdf521bccd63218a333ec82a82851e02945b7f63f58dcf07073", + "992b757e11570bdf521bccd63218a333ec82a82851e02945b7f63f58dcf07073", + "992b757e11570bdf521bccd63218a333ec82a82851e02945b7f63f58dcf07073" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_regex": "\\d{4}-\\d{2}-\\d{2}", + "max_tokens": 400, + "messages": [ + { + "content": "When did the first person walk on the moon? Give the date.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "613ee0089978b70c8c5f13ea0b1411793e8e2e09a3d03fdc84b0b5bb28aa14e8", + "613ee0089978b70c8c5f13ea0b1411793e8e2e09a3d03fdc84b0b5bb28aa14e8", + "613ee0089978b70c8c5f13ea0b1411793e8e2e09a3d03fdc84b0b5bb28aa14e8" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_regex": "\\d{4}-\\d{2}-\\d{2}", + "max_tokens": 1200, + "messages": [ + { + "content": "When did the first person walk on the moon? Give the date.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "d8c9b5ca8545ff2672e78d1bec92b38bc06719f8171049359250d4358e71e56f", + "d8c9b5ca8545ff2672e78d1bec92b38bc06719f8171049359250d4358e71e56f", + "d8c9b5ca8545ff2672e78d1bec92b38bc06719f8171049359250d4358e71e56f" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_regex": "\\d{4}-\\d{2}-\\d{2}", + "max_tokens": 400, + "messages": [ + { + "content": "When did the first person walk on the moon? Give the date.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "94ace2a871dda93e340c3a2c4501cd061198a7e5ac82b6f32ce15e1a7be2ab7d", + "94ace2a871dda93e340c3a2c4501cd061198a7e5ac82b6f32ce15e1a7be2ab7d", + "94ace2a871dda93e340c3a2c4501cd061198a7e5ac82b6f32ce15e1a7be2ab7d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_regex": "\\d{4}-\\d{2}-\\d{2}", + "max_tokens": 1200, + "messages": [ + { + "content": "When did the first person walk on the moon? Give the date.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "73317173791629306890fe4163ae8ef1b112f66164430be5cf413b20a8d6668d", + "73317173791629306890fe4163ae8ef1b112f66164430be5cf413b20a8d6668d", + "73317173791629306890fe4163ae8ef1b112f66164430be5cf413b20a8d6668d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_choice": [ + "positive", + "negative", + "neutral" + ], + "max_tokens": 400, + "messages": [ + { + "content": "Classify the sentiment: 'I loved every minute of this film.'", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d", + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d", + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_choice": [ + "positive", + "negative", + "neutral" + ], + "max_tokens": 1200, + "messages": [ + { + "content": "Classify the sentiment: 'I loved every minute of this film.'", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "1e17a6da44a8ae08e848f819966f156cf9556f85540814634c8bdbedc65a30ae", + "1e17a6da44a8ae08e848f819966f156cf9556f85540814634c8bdbedc65a30ae", + "1e17a6da44a8ae08e848f819966f156cf9556f85540814634c8bdbedc65a30ae" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_choice": [ + "positive", + "negative", + "neutral" + ], + "max_tokens": 400, + "messages": [ + { + "content": "Classify the sentiment: 'I loved every minute of this film.'", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d", + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d", + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_choice": [ + "positive", + "negative", + "neutral" + ], + "max_tokens": 1200, + "messages": [ + { + "content": "Classify the sentiment: 'I loved every minute of this film.'", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "7d1aa328f84f06ec6e56421145b066954dcab17d74a2bb222271a610419fbc15", + "7d1aa328f84f06ec6e56421145b066954dcab17d74a2bb222271a610419fbc15", + "7d1aa328f84f06ec6e56421145b066954dcab17d74a2bb222271a610419fbc15" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_grammar": "root ::= \"Answer: \" (\"yes\" | \"no\") \", because \" [a-z ]{3,40} \".\"", + "max_tokens": 400, + "messages": [ + { + "content": "Is the sky blue on a clear day?", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "321c709fab0bb0a79580785b993f8b74105bdd54fdf9a2e3a6add56d298b4bf5", + "321c709fab0bb0a79580785b993f8b74105bdd54fdf9a2e3a6add56d298b4bf5", + "321c709fab0bb0a79580785b993f8b74105bdd54fdf9a2e3a6add56d298b4bf5" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_grammar": "root ::= \"Answer: \" (\"yes\" | \"no\") \", because \" [a-z ]{3,40} \".\"", + "max_tokens": 1200, + "messages": [ + { + "content": "Is the sky blue on a clear day?", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "67d2a38bf0777d6c703cae6bd95ae4fac325304dab7da7c75201db1e235e359e", + "67d2a38bf0777d6c703cae6bd95ae4fac325304dab7da7c75201db1e235e359e", + "67d2a38bf0777d6c703cae6bd95ae4fac325304dab7da7c75201db1e235e359e" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_grammar": "root ::= \"Answer: \" (\"yes\" | \"no\") \", because \" [a-z ]{3,40} \".\"", + "max_tokens": 400, + "messages": [ + { + "content": "Is the sky blue on a clear day?", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "b0e4ab03e7c58b4c136ee29584be22fa1d6fe701e5a6da7ed7079264119d1e29", + "b0e4ab03e7c58b4c136ee29584be22fa1d6fe701e5a6da7ed7079264119d1e29", + "b0e4ab03e7c58b4c136ee29584be22fa1d6fe701e5a6da7ed7079264119d1e29" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_grammar": "root ::= \"Answer: \" (\"yes\" | \"no\") \", because \" [a-z ]{3,40} \".\"", + "max_tokens": 1200, + "messages": [ + { + "content": "Is the sky blue on a clear day?", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "8244dc9f6a9fb711a721dc8fecf31fe515a693e5ff16e5c660f9a44870f874a1", + "8244dc9f6a9fb711a721dc8fecf31fe515a693e5ff16e5c660f9a44870f874a1", + "8244dc9f6a9fb711a721dc8fecf31fe515a693e5ff16e5c660f9a44870f874a1" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Give today's date in ISO form.", + "role": "user" + } + ], + "model": "lfm3-check", + "structured_outputs": { + "regex": "\\d{4}-\\d{2}-\\d{2}" + }, + "temperature": 0 + }, + "count": 3, + "hashes": [ + "64e4286600a8662814240ae2b715c6adce7a68df0aa3a1c9a7c275ffb55f39cc", + "64e4286600a8662814240ae2b715c6adce7a68df0aa3a1c9a7c275ffb55f39cc", + "64e4286600a8662814240ae2b715c6adce7a68df0aa3a1c9a7c275ffb55f39cc" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Give today's date in ISO form.", + "role": "user" + } + ], + "model": "lfm3-check", + "structured_outputs": { + "regex": "\\d{4}-\\d{2}-\\d{2}" + }, + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "400899839ae62d552c2ae9a929b64796294a2ccd36cddad35c9bee8838dc8016", + "400899839ae62d552c2ae9a929b64796294a2ccd36cddad35c9bee8838dc8016", + "400899839ae62d552c2ae9a929b64796294a2ccd36cddad35c9bee8838dc8016" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Give today's date in ISO form.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "structured_outputs": { + "regex": "\\d{4}-\\d{2}-\\d{2}" + }, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "64e4286600a8662814240ae2b715c6adce7a68df0aa3a1c9a7c275ffb55f39cc", + "64e4286600a8662814240ae2b715c6adce7a68df0aa3a1c9a7c275ffb55f39cc", + "64e4286600a8662814240ae2b715c6adce7a68df0aa3a1c9a7c275ffb55f39cc" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Give today's date in ISO form.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "structured_outputs": { + "regex": "\\d{4}-\\d{2}-\\d{2}" + }, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "bc669b4659e66c9409fd9889cfcac61d53e440326d1b81220e9dc68f4780b56e", + "bc669b4659e66c9409fd9889cfcac61d53e440326d1b81220e9dc68f4780b56e", + "bc669b4659e66c9409fd9889cfcac61d53e440326d1b81220e9dc68f4780b56e" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "71bea47b2367b433e957cc461a2a8150a0a7c0e73ae724381a2420078d80011d", + "71bea47b2367b433e957cc461a2a8150a0a7c0e73ae724381a2420078d80011d", + "71bea47b2367b433e957cc461a2a8150a0a7c0e73ae724381a2420078d80011d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "625ea27de49c9d2833132231eff66fb71c9e6062983e6c13eb0c24220cb1afc1", + "625ea27de49c9d2833132231eff66fb71c9e6062983e6c13eb0c24220cb1afc1", + "625ea27de49c9d2833132231eff66fb71c9e6062983e6c13eb0c24220cb1afc1" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "90528efb4df26e34f648b00717102f32113c2b363224cbac56b8c3b8ff4768ff", + "90528efb4df26e34f648b00717102f32113c2b363224cbac56b8c3b8ff4768ff", + "90528efb4df26e34f648b00717102f32113c2b363224cbac56b8c3b8ff4768ff" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "5eaa966d72c48c19e4f5f1d99bbf48d937692e3311587041e7f6d8d604e61494", + "5eaa966d72c48c19e4f5f1d99bbf48d937692e3311587041e7f6d8d604e61494", + "5eaa966d72c48c19e4f5f1d99bbf48d937692e3311587041e7f6d8d604e61494" + ], + "verified": true + } + ] + }, + { + "platform": "M4 Pro, macOS 26.5.1", + "checkpoint": "VL-2B-4", + "native_binary_sha256": "5c8a04d4af599879797f4f5bed8e00ab7d9d70732e698bd2a1ec2a01ec762e09", + "original_client_exit": 0, + "groups": 32, + "passed_full_token_equality": 32, + "failed_full_token_equality": 0, + "checks": [ + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "temperature": 0 + }, + "count": 3, + "hashes": [ + "b5c5f176466925a1dc9de8fff4578df4eb4e0efcdac74df82d5df7189f9de5cc", + "b5c5f176466925a1dc9de8fff4578df4eb4e0efcdac74df82d5df7189f9de5cc", + "b5c5f176466925a1dc9de8fff4578df4eb4e0efcdac74df82d5df7189f9de5cc" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "c41981cafaa3da59c92b9e34ce1fd12f59afc0a0fb7229b849c05caf8288a35e", + "c41981cafaa3da59c92b9e34ce1fd12f59afc0a0fb7229b849c05caf8288a35e", + "c41981cafaa3da59c92b9e34ce1fd12f59afc0a0fb7229b849c05caf8288a35e" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "caec2cb7683056a6abc4a97e4f6d632aa9a4efab60bc84a4240224c93168b289", + "caec2cb7683056a6abc4a97e4f6d632aa9a4efab60bc84a4240224c93168b289", + "caec2cb7683056a6abc4a97e4f6d632aa9a4efab60bc84a4240224c93168b289" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "5fa4062e683e5f4dbc6acb92975c12d3968d4e6da37be52f0d14e4393f2d277c", + "5fa4062e683e5f4dbc6acb92975c12d3968d4e6da37be52f0d14e4393f2d277c", + "5fa4062e683e5f4dbc6acb92975c12d3968d4e6da37be52f0d14e4393f2d277c" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "What is the weather like in Lisbon today? Answer as JSON.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "json_schema": { + "name": "weather", + "schema": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + } + }, + "type": "json_schema" + }, + "temperature": 0 + }, + "count": 3, + "hashes": [ + "517656ddfd0a502e0294104fe71d34a05302b61498a49da054a68987fa455812", + "517656ddfd0a502e0294104fe71d34a05302b61498a49da054a68987fa455812", + "517656ddfd0a502e0294104fe71d34a05302b61498a49da054a68987fa455812" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "What is the weather like in Lisbon today? Answer as JSON.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "json_schema": { + "name": "weather", + "schema": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + } + }, + "type": "json_schema" + }, + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "361edec20fd0e7e737574dfaeec3d290f52a41f6b390a4a05fd488a0be413bf4", + "361edec20fd0e7e737574dfaeec3d290f52a41f6b390a4a05fd488a0be413bf4", + "361edec20fd0e7e737574dfaeec3d290f52a41f6b390a4a05fd488a0be413bf4" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "What is the weather like in Lisbon today? Answer as JSON.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "json_schema": { + "name": "weather", + "schema": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + } + }, + "type": "json_schema" + }, + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "a829538ceb2ceb71462719b460824cfc6abc31b4bbf089ac80266aad3998be70", + "a829538ceb2ceb71462719b460824cfc6abc31b4bbf089ac80266aad3998be70", + "a829538ceb2ceb71462719b460824cfc6abc31b4bbf089ac80266aad3998be70" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "What is the weather like in Lisbon today? Answer as JSON.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "json_schema": { + "name": "weather", + "schema": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + } + }, + "type": "json_schema" + }, + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "e334d523adbfbe07941af4f2efd60ba0955b74aa7ef7ff0d955dbaef5b2d684f", + "e334d523adbfbe07941af4f2efd60ba0955b74aa7ef7ff0d955dbaef5b2d684f", + "e334d523adbfbe07941af4f2efd60ba0955b74aa7ef7ff0d955dbaef5b2d684f" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_json": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + }, + "max_tokens": 400, + "messages": [ + { + "content": "Make up the weather in Oslo.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "ec0a75ef9736b0eaa4ddd2bb5ed64eb32080b2ecbcd9dfafa72ad32675eb0784", + "ec0a75ef9736b0eaa4ddd2bb5ed64eb32080b2ecbcd9dfafa72ad32675eb0784", + "ec0a75ef9736b0eaa4ddd2bb5ed64eb32080b2ecbcd9dfafa72ad32675eb0784" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_json": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Make up the weather in Oslo.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "342863a4b621d4d5c3b51d20415c53867cb35165f8fb08fd5d03e53d5661486f", + "342863a4b621d4d5c3b51d20415c53867cb35165f8fb08fd5d03e53d5661486f", + "342863a4b621d4d5c3b51d20415c53867cb35165f8fb08fd5d03e53d5661486f" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_json": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + }, + "max_tokens": 400, + "messages": [ + { + "content": "Make up the weather in Oslo.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "ffd631b0cc4d82afddc08ebb896fb32419da3f546dda4854762093a24b077f37", + "ffd631b0cc4d82afddc08ebb896fb32419da3f546dda4854762093a24b077f37", + "ffd631b0cc4d82afddc08ebb896fb32419da3f546dda4854762093a24b077f37" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_json": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Make up the weather in Oslo.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "858bc1207e3f40bac2597de755d39a6f66f82378691c31a78c565d7c3e30931a", + "858bc1207e3f40bac2597de755d39a6f66f82378691c31a78c565d7c3e30931a", + "858bc1207e3f40bac2597de755d39a6f66f82378691c31a78c565d7c3e30931a" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_regex": "\\d{4}-\\d{2}-\\d{2}", + "max_tokens": 400, + "messages": [ + { + "content": "When did the first person walk on the moon? Give the date.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "5ad44b55a70b073ffc05cd0d5da67f2d8d7ec609ce3ed820534ed69697ddbe76", + "5ad44b55a70b073ffc05cd0d5da67f2d8d7ec609ce3ed820534ed69697ddbe76", + "5ad44b55a70b073ffc05cd0d5da67f2d8d7ec609ce3ed820534ed69697ddbe76" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_regex": "\\d{4}-\\d{2}-\\d{2}", + "max_tokens": 1200, + "messages": [ + { + "content": "When did the first person walk on the moon? Give the date.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "508f46ee87de098b28863ee385c865240bfc57c9b1b794fa81390c0958659825", + "508f46ee87de098b28863ee385c865240bfc57c9b1b794fa81390c0958659825", + "508f46ee87de098b28863ee385c865240bfc57c9b1b794fa81390c0958659825" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_regex": "\\d{4}-\\d{2}-\\d{2}", + "max_tokens": 400, + "messages": [ + { + "content": "When did the first person walk on the moon? Give the date.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "466faf974ed2633c54f51bf9055f8c7f39c9e6a3fc9f759be47d5fe6f2521291", + "466faf974ed2633c54f51bf9055f8c7f39c9e6a3fc9f759be47d5fe6f2521291", + "466faf974ed2633c54f51bf9055f8c7f39c9e6a3fc9f759be47d5fe6f2521291" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_regex": "\\d{4}-\\d{2}-\\d{2}", + "max_tokens": 1200, + "messages": [ + { + "content": "When did the first person walk on the moon? Give the date.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "028e6b85824bac4cd03f1572b22f667f02bd10ff94be88cbf9b6cfceadcc9892", + "028e6b85824bac4cd03f1572b22f667f02bd10ff94be88cbf9b6cfceadcc9892", + "028e6b85824bac4cd03f1572b22f667f02bd10ff94be88cbf9b6cfceadcc9892" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_choice": [ + "positive", + "negative", + "neutral" + ], + "max_tokens": 400, + "messages": [ + { + "content": "Classify the sentiment: 'I loved every minute of this film.'", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d", + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d", + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_choice": [ + "positive", + "negative", + "neutral" + ], + "max_tokens": 1200, + "messages": [ + { + "content": "Classify the sentiment: 'I loved every minute of this film.'", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "896312515f03752a513f67ba40cb98b81b6212f4141dc783d93fcda3eada7b11", + "896312515f03752a513f67ba40cb98b81b6212f4141dc783d93fcda3eada7b11", + "896312515f03752a513f67ba40cb98b81b6212f4141dc783d93fcda3eada7b11" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_choice": [ + "positive", + "negative", + "neutral" + ], + "max_tokens": 400, + "messages": [ + { + "content": "Classify the sentiment: 'I loved every minute of this film.'", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d", + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d", + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_choice": [ + "positive", + "negative", + "neutral" + ], + "max_tokens": 1200, + "messages": [ + { + "content": "Classify the sentiment: 'I loved every minute of this film.'", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "3e09b96b2542053ab74ead9d3b377a78936bf8f2a4775c5f2a61f31076f6bf04", + "3e09b96b2542053ab74ead9d3b377a78936bf8f2a4775c5f2a61f31076f6bf04", + "3e09b96b2542053ab74ead9d3b377a78936bf8f2a4775c5f2a61f31076f6bf04" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_grammar": "root ::= \"Answer: \" (\"yes\" | \"no\") \", because \" [a-z ]{3,40} \".\"", + "max_tokens": 400, + "messages": [ + { + "content": "Is the sky blue on a clear day?", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "1c4b25c4b3154630f336ea4e5c3f8029fab55a0a171c7b53a70ddfb5c9f5ab04", + "1c4b25c4b3154630f336ea4e5c3f8029fab55a0a171c7b53a70ddfb5c9f5ab04", + "1c4b25c4b3154630f336ea4e5c3f8029fab55a0a171c7b53a70ddfb5c9f5ab04" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_grammar": "root ::= \"Answer: \" (\"yes\" | \"no\") \", because \" [a-z ]{3,40} \".\"", + "max_tokens": 1200, + "messages": [ + { + "content": "Is the sky blue on a clear day?", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "352433279b31016172860966480593c25abd4083021e0ad2b2f4510a69e3f837", + "352433279b31016172860966480593c25abd4083021e0ad2b2f4510a69e3f837", + "352433279b31016172860966480593c25abd4083021e0ad2b2f4510a69e3f837" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_grammar": "root ::= \"Answer: \" (\"yes\" | \"no\") \", because \" [a-z ]{3,40} \".\"", + "max_tokens": 400, + "messages": [ + { + "content": "Is the sky blue on a clear day?", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "ba64d4389e5d82b0a3e4119e04d1d99432567b6582c8e23e5db3a64ed5761415", + "ba64d4389e5d82b0a3e4119e04d1d99432567b6582c8e23e5db3a64ed5761415", + "ba64d4389e5d82b0a3e4119e04d1d99432567b6582c8e23e5db3a64ed5761415" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_grammar": "root ::= \"Answer: \" (\"yes\" | \"no\") \", because \" [a-z ]{3,40} \".\"", + "max_tokens": 1200, + "messages": [ + { + "content": "Is the sky blue on a clear day?", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "fbe0f857a5b704d9b64254e165b8384cbb7a16f2108e4b5e4d9bdfde8ea7ee8d", + "fbe0f857a5b704d9b64254e165b8384cbb7a16f2108e4b5e4d9bdfde8ea7ee8d", + "fbe0f857a5b704d9b64254e165b8384cbb7a16f2108e4b5e4d9bdfde8ea7ee8d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Give today's date in ISO form.", + "role": "user" + } + ], + "model": "lfm3-check", + "structured_outputs": { + "regex": "\\d{4}-\\d{2}-\\d{2}" + }, + "temperature": 0 + }, + "count": 3, + "hashes": [ + "a4397e998056b0d725b6151df114c2a4c916e9efce98a20f11ad6666c272bb39", + "a4397e998056b0d725b6151df114c2a4c916e9efce98a20f11ad6666c272bb39", + "a4397e998056b0d725b6151df114c2a4c916e9efce98a20f11ad6666c272bb39" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Give today's date in ISO form.", + "role": "user" + } + ], + "model": "lfm3-check", + "structured_outputs": { + "regex": "\\d{4}-\\d{2}-\\d{2}" + }, + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "3daa2e15ab32f03e3ff68a8c4d72b7fdc7662fb4eca448d9960644cac8251c1e", + "3daa2e15ab32f03e3ff68a8c4d72b7fdc7662fb4eca448d9960644cac8251c1e", + "3daa2e15ab32f03e3ff68a8c4d72b7fdc7662fb4eca448d9960644cac8251c1e" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Give today's date in ISO form.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "structured_outputs": { + "regex": "\\d{4}-\\d{2}-\\d{2}" + }, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "12c1bc9f4e0160d763f9ecedc28e39f81b66664c598cacdd60d3739678fefe81", + "12c1bc9f4e0160d763f9ecedc28e39f81b66664c598cacdd60d3739678fefe81", + "12c1bc9f4e0160d763f9ecedc28e39f81b66664c598cacdd60d3739678fefe81" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Give today's date in ISO form.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "structured_outputs": { + "regex": "\\d{4}-\\d{2}-\\d{2}" + }, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "9dd396aa81f2a111491d2ccf92bd6674845adabc14a3e295e4266ec50c63556f", + "9dd396aa81f2a111491d2ccf92bd6674845adabc14a3e295e4266ec50c63556f", + "9dd396aa81f2a111491d2ccf92bd6674845adabc14a3e295e4266ec50c63556f" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "3594736c0c8b914daa5abd7a250ef25e43dd0818d86ab57e51c6acf8360c2144", + "3594736c0c8b914daa5abd7a250ef25e43dd0818d86ab57e51c6acf8360c2144", + "3594736c0c8b914daa5abd7a250ef25e43dd0818d86ab57e51c6acf8360c2144" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "d84edd56121bbadfbd11fd5e3a5d6e048aae30d4de5152be87d8608f18011260", + "d84edd56121bbadfbd11fd5e3a5d6e048aae30d4de5152be87d8608f18011260", + "d84edd56121bbadfbd11fd5e3a5d6e048aae30d4de5152be87d8608f18011260" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "3594736c0c8b914daa5abd7a250ef25e43dd0818d86ab57e51c6acf8360c2144", + "3594736c0c8b914daa5abd7a250ef25e43dd0818d86ab57e51c6acf8360c2144", + "3594736c0c8b914daa5abd7a250ef25e43dd0818d86ab57e51c6acf8360c2144" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "91a1e14a5b9fc3e27a8b4821548c7203af467f0ca1fdea0b4ba3156a2dd27a49", + "91a1e14a5b9fc3e27a8b4821548c7203af467f0ca1fdea0b4ba3156a2dd27a49", + "91a1e14a5b9fc3e27a8b4821548c7203af467f0ca1fdea0b4ba3156a2dd27a49" + ], + "verified": true + } + ] + }, + { + "platform": "M5 Ultra, macOS 27.0", + "checkpoint": "1.7B-4", + "native_binary_sha256": "251079309919aa9b21ca8cc6a89c58fc3fc9a9baf800fa5ce6a6af313a5b5f65", + "original_client_exit": 0, + "groups": 32, + "passed_full_token_equality": 32, + "failed_full_token_equality": 0, + "checks": [ + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "temperature": 0 + }, + "count": 3, + "hashes": [ + "97fc568e1c13add8c4f1c58c38ab18a87039408de0d789a92614693a8f5c73d0", + "97fc568e1c13add8c4f1c58c38ab18a87039408de0d789a92614693a8f5c73d0", + "97fc568e1c13add8c4f1c58c38ab18a87039408de0d789a92614693a8f5c73d0" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "8f3cef2f2dd40372a9f5a5a45465d005711e83e73de634020964ea1164f102fc", + "8f3cef2f2dd40372a9f5a5a45465d005711e83e73de634020964ea1164f102fc", + "8f3cef2f2dd40372a9f5a5a45465d005711e83e73de634020964ea1164f102fc" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "28e9992d945cd22c289c5a1604e9d84b8fd4d93908d1d3538e441728c2d4b149", + "28e9992d945cd22c289c5a1604e9d84b8fd4d93908d1d3538e441728c2d4b149", + "28e9992d945cd22c289c5a1604e9d84b8fd4d93908d1d3538e441728c2d4b149" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "cda680f6cc3d90448f12337ac2cc1b0119f16da48a71ace0e8e1f192c9945a6d", + "cda680f6cc3d90448f12337ac2cc1b0119f16da48a71ace0e8e1f192c9945a6d", + "cda680f6cc3d90448f12337ac2cc1b0119f16da48a71ace0e8e1f192c9945a6d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "What is the weather like in Lisbon today? Answer as JSON.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "json_schema": { + "name": "weather", + "schema": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + } + }, + "type": "json_schema" + }, + "temperature": 0 + }, + "count": 3, + "hashes": [ + "d455a572b3e0b29505b67549d22ee6dbdaa3453a3f3d4c4a74d5f9c53c288ac7", + "d455a572b3e0b29505b67549d22ee6dbdaa3453a3f3d4c4a74d5f9c53c288ac7", + "d455a572b3e0b29505b67549d22ee6dbdaa3453a3f3d4c4a74d5f9c53c288ac7" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "What is the weather like in Lisbon today? Answer as JSON.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "json_schema": { + "name": "weather", + "schema": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + } + }, + "type": "json_schema" + }, + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "a6703c6b4506a475098af2e37ca37cc66f508ffd46238e5f76546b1ed795021d", + "a6703c6b4506a475098af2e37ca37cc66f508ffd46238e5f76546b1ed795021d", + "a6703c6b4506a475098af2e37ca37cc66f508ffd46238e5f76546b1ed795021d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "What is the weather like in Lisbon today? Answer as JSON.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "json_schema": { + "name": "weather", + "schema": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + } + }, + "type": "json_schema" + }, + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "e2ced68110d55bbdfede82e2431832288dc53c5f2ac44e0f99e4cf28ada34b1c", + "e2ced68110d55bbdfede82e2431832288dc53c5f2ac44e0f99e4cf28ada34b1c", + "e2ced68110d55bbdfede82e2431832288dc53c5f2ac44e0f99e4cf28ada34b1c" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "What is the weather like in Lisbon today? Answer as JSON.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "json_schema": { + "name": "weather", + "schema": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + } + }, + "type": "json_schema" + }, + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "585b2f3c6566d27343b7af1c4464f280c389a54f26ed878de701db5c0705a147", + "585b2f3c6566d27343b7af1c4464f280c389a54f26ed878de701db5c0705a147", + "585b2f3c6566d27343b7af1c4464f280c389a54f26ed878de701db5c0705a147" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_json": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + }, + "max_tokens": 400, + "messages": [ + { + "content": "Make up the weather in Oslo.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "d2bf435d203b8e78856bb7eae3e2e7b7c31ae53d883fcf6564fb58f1a938376c", + "d2bf435d203b8e78856bb7eae3e2e7b7c31ae53d883fcf6564fb58f1a938376c", + "d2bf435d203b8e78856bb7eae3e2e7b7c31ae53d883fcf6564fb58f1a938376c" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_json": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Make up the weather in Oslo.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "2febc228486dea3590e75587cb09af791e2c289b3f1f589ba69b4d81100c390d", + "2febc228486dea3590e75587cb09af791e2c289b3f1f589ba69b4d81100c390d", + "2febc228486dea3590e75587cb09af791e2c289b3f1f589ba69b4d81100c390d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_json": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + }, + "max_tokens": 400, + "messages": [ + { + "content": "Make up the weather in Oslo.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "4faffa1eba4aecc775cd93a9b87f1189e4df90760c98d364a69b9b0b07223162", + "4faffa1eba4aecc775cd93a9b87f1189e4df90760c98d364a69b9b0b07223162", + "4faffa1eba4aecc775cd93a9b87f1189e4df90760c98d364a69b9b0b07223162" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_json": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Make up the weather in Oslo.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "992b757e11570bdf521bccd63218a333ec82a82851e02945b7f63f58dcf07073", + "992b757e11570bdf521bccd63218a333ec82a82851e02945b7f63f58dcf07073", + "992b757e11570bdf521bccd63218a333ec82a82851e02945b7f63f58dcf07073" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_regex": "\\d{4}-\\d{2}-\\d{2}", + "max_tokens": 400, + "messages": [ + { + "content": "When did the first person walk on the moon? Give the date.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "613ee0089978b70c8c5f13ea0b1411793e8e2e09a3d03fdc84b0b5bb28aa14e8", + "613ee0089978b70c8c5f13ea0b1411793e8e2e09a3d03fdc84b0b5bb28aa14e8", + "613ee0089978b70c8c5f13ea0b1411793e8e2e09a3d03fdc84b0b5bb28aa14e8" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_regex": "\\d{4}-\\d{2}-\\d{2}", + "max_tokens": 1200, + "messages": [ + { + "content": "When did the first person walk on the moon? Give the date.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "d8c9b5ca8545ff2672e78d1bec92b38bc06719f8171049359250d4358e71e56f", + "d8c9b5ca8545ff2672e78d1bec92b38bc06719f8171049359250d4358e71e56f", + "d8c9b5ca8545ff2672e78d1bec92b38bc06719f8171049359250d4358e71e56f" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_regex": "\\d{4}-\\d{2}-\\d{2}", + "max_tokens": 400, + "messages": [ + { + "content": "When did the first person walk on the moon? Give the date.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "94ace2a871dda93e340c3a2c4501cd061198a7e5ac82b6f32ce15e1a7be2ab7d", + "94ace2a871dda93e340c3a2c4501cd061198a7e5ac82b6f32ce15e1a7be2ab7d", + "94ace2a871dda93e340c3a2c4501cd061198a7e5ac82b6f32ce15e1a7be2ab7d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_regex": "\\d{4}-\\d{2}-\\d{2}", + "max_tokens": 1200, + "messages": [ + { + "content": "When did the first person walk on the moon? Give the date.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "73317173791629306890fe4163ae8ef1b112f66164430be5cf413b20a8d6668d", + "73317173791629306890fe4163ae8ef1b112f66164430be5cf413b20a8d6668d", + "73317173791629306890fe4163ae8ef1b112f66164430be5cf413b20a8d6668d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_choice": [ + "positive", + "negative", + "neutral" + ], + "max_tokens": 400, + "messages": [ + { + "content": "Classify the sentiment: 'I loved every minute of this film.'", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d", + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d", + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_choice": [ + "positive", + "negative", + "neutral" + ], + "max_tokens": 1200, + "messages": [ + { + "content": "Classify the sentiment: 'I loved every minute of this film.'", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "1e17a6da44a8ae08e848f819966f156cf9556f85540814634c8bdbedc65a30ae", + "1e17a6da44a8ae08e848f819966f156cf9556f85540814634c8bdbedc65a30ae", + "1e17a6da44a8ae08e848f819966f156cf9556f85540814634c8bdbedc65a30ae" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_choice": [ + "positive", + "negative", + "neutral" + ], + "max_tokens": 400, + "messages": [ + { + "content": "Classify the sentiment: 'I loved every minute of this film.'", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d", + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d", + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_choice": [ + "positive", + "negative", + "neutral" + ], + "max_tokens": 1200, + "messages": [ + { + "content": "Classify the sentiment: 'I loved every minute of this film.'", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "7d1aa328f84f06ec6e56421145b066954dcab17d74a2bb222271a610419fbc15", + "7d1aa328f84f06ec6e56421145b066954dcab17d74a2bb222271a610419fbc15", + "7d1aa328f84f06ec6e56421145b066954dcab17d74a2bb222271a610419fbc15" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_grammar": "root ::= \"Answer: \" (\"yes\" | \"no\") \", because \" [a-z ]{3,40} \".\"", + "max_tokens": 400, + "messages": [ + { + "content": "Is the sky blue on a clear day?", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "321c709fab0bb0a79580785b993f8b74105bdd54fdf9a2e3a6add56d298b4bf5", + "321c709fab0bb0a79580785b993f8b74105bdd54fdf9a2e3a6add56d298b4bf5", + "321c709fab0bb0a79580785b993f8b74105bdd54fdf9a2e3a6add56d298b4bf5" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_grammar": "root ::= \"Answer: \" (\"yes\" | \"no\") \", because \" [a-z ]{3,40} \".\"", + "max_tokens": 1200, + "messages": [ + { + "content": "Is the sky blue on a clear day?", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "67d2a38bf0777d6c703cae6bd95ae4fac325304dab7da7c75201db1e235e359e", + "67d2a38bf0777d6c703cae6bd95ae4fac325304dab7da7c75201db1e235e359e", + "67d2a38bf0777d6c703cae6bd95ae4fac325304dab7da7c75201db1e235e359e" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_grammar": "root ::= \"Answer: \" (\"yes\" | \"no\") \", because \" [a-z ]{3,40} \".\"", + "max_tokens": 400, + "messages": [ + { + "content": "Is the sky blue on a clear day?", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "b0e4ab03e7c58b4c136ee29584be22fa1d6fe701e5a6da7ed7079264119d1e29", + "b0e4ab03e7c58b4c136ee29584be22fa1d6fe701e5a6da7ed7079264119d1e29", + "b0e4ab03e7c58b4c136ee29584be22fa1d6fe701e5a6da7ed7079264119d1e29" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_grammar": "root ::= \"Answer: \" (\"yes\" | \"no\") \", because \" [a-z ]{3,40} \".\"", + "max_tokens": 1200, + "messages": [ + { + "content": "Is the sky blue on a clear day?", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "8244dc9f6a9fb711a721dc8fecf31fe515a693e5ff16e5c660f9a44870f874a1", + "8244dc9f6a9fb711a721dc8fecf31fe515a693e5ff16e5c660f9a44870f874a1", + "8244dc9f6a9fb711a721dc8fecf31fe515a693e5ff16e5c660f9a44870f874a1" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Give today's date in ISO form.", + "role": "user" + } + ], + "model": "lfm3-check", + "structured_outputs": { + "regex": "\\d{4}-\\d{2}-\\d{2}" + }, + "temperature": 0 + }, + "count": 3, + "hashes": [ + "64e4286600a8662814240ae2b715c6adce7a68df0aa3a1c9a7c275ffb55f39cc", + "64e4286600a8662814240ae2b715c6adce7a68df0aa3a1c9a7c275ffb55f39cc", + "64e4286600a8662814240ae2b715c6adce7a68df0aa3a1c9a7c275ffb55f39cc" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Give today's date in ISO form.", + "role": "user" + } + ], + "model": "lfm3-check", + "structured_outputs": { + "regex": "\\d{4}-\\d{2}-\\d{2}" + }, + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "400899839ae62d552c2ae9a929b64796294a2ccd36cddad35c9bee8838dc8016", + "400899839ae62d552c2ae9a929b64796294a2ccd36cddad35c9bee8838dc8016", + "400899839ae62d552c2ae9a929b64796294a2ccd36cddad35c9bee8838dc8016" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Give today's date in ISO form.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "structured_outputs": { + "regex": "\\d{4}-\\d{2}-\\d{2}" + }, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "64e4286600a8662814240ae2b715c6adce7a68df0aa3a1c9a7c275ffb55f39cc", + "64e4286600a8662814240ae2b715c6adce7a68df0aa3a1c9a7c275ffb55f39cc", + "64e4286600a8662814240ae2b715c6adce7a68df0aa3a1c9a7c275ffb55f39cc" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Give today's date in ISO form.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "structured_outputs": { + "regex": "\\d{4}-\\d{2}-\\d{2}" + }, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "bc669b4659e66c9409fd9889cfcac61d53e440326d1b81220e9dc68f4780b56e", + "bc669b4659e66c9409fd9889cfcac61d53e440326d1b81220e9dc68f4780b56e", + "bc669b4659e66c9409fd9889cfcac61d53e440326d1b81220e9dc68f4780b56e" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "71bea47b2367b433e957cc461a2a8150a0a7c0e73ae724381a2420078d80011d", + "71bea47b2367b433e957cc461a2a8150a0a7c0e73ae724381a2420078d80011d", + "71bea47b2367b433e957cc461a2a8150a0a7c0e73ae724381a2420078d80011d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "625ea27de49c9d2833132231eff66fb71c9e6062983e6c13eb0c24220cb1afc1", + "625ea27de49c9d2833132231eff66fb71c9e6062983e6c13eb0c24220cb1afc1", + "625ea27de49c9d2833132231eff66fb71c9e6062983e6c13eb0c24220cb1afc1" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "90528efb4df26e34f648b00717102f32113c2b363224cbac56b8c3b8ff4768ff", + "90528efb4df26e34f648b00717102f32113c2b363224cbac56b8c3b8ff4768ff", + "90528efb4df26e34f648b00717102f32113c2b363224cbac56b8c3b8ff4768ff" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "5eaa966d72c48c19e4f5f1d99bbf48d937692e3311587041e7f6d8d604e61494", + "5eaa966d72c48c19e4f5f1d99bbf48d937692e3311587041e7f6d8d604e61494", + "5eaa966d72c48c19e4f5f1d99bbf48d937692e3311587041e7f6d8d604e61494" + ], + "verified": true + } + ] + }, + { + "platform": "M5 Ultra, macOS 27.0", + "checkpoint": "VL-2B-4", + "native_binary_sha256": "251079309919aa9b21ca8cc6a89c58fc3fc9a9baf800fa5ce6a6af313a5b5f65", + "original_client_exit": 0, + "groups": 32, + "passed_full_token_equality": 32, + "failed_full_token_equality": 0, + "checks": [ + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "temperature": 0 + }, + "count": 3, + "hashes": [ + "b5c5f176466925a1dc9de8fff4578df4eb4e0efcdac74df82d5df7189f9de5cc", + "b5c5f176466925a1dc9de8fff4578df4eb4e0efcdac74df82d5df7189f9de5cc", + "b5c5f176466925a1dc9de8fff4578df4eb4e0efcdac74df82d5df7189f9de5cc" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "c41981cafaa3da59c92b9e34ce1fd12f59afc0a0fb7229b849c05caf8288a35e", + "c41981cafaa3da59c92b9e34ce1fd12f59afc0a0fb7229b849c05caf8288a35e", + "c41981cafaa3da59c92b9e34ce1fd12f59afc0a0fb7229b849c05caf8288a35e" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "caec2cb7683056a6abc4a97e4f6d632aa9a4efab60bc84a4240224c93168b289", + "caec2cb7683056a6abc4a97e4f6d632aa9a4efab60bc84a4240224c93168b289", + "caec2cb7683056a6abc4a97e4f6d632aa9a4efab60bc84a4240224c93168b289" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Describe a cat as a JSON object with a few fields.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "type": "json_object" + }, + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "5fa4062e683e5f4dbc6acb92975c12d3968d4e6da37be52f0d14e4393f2d277c", + "5fa4062e683e5f4dbc6acb92975c12d3968d4e6da37be52f0d14e4393f2d277c", + "5fa4062e683e5f4dbc6acb92975c12d3968d4e6da37be52f0d14e4393f2d277c" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "What is the weather like in Lisbon today? Answer as JSON.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "json_schema": { + "name": "weather", + "schema": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + } + }, + "type": "json_schema" + }, + "temperature": 0 + }, + "count": 3, + "hashes": [ + "517656ddfd0a502e0294104fe71d34a05302b61498a49da054a68987fa455812", + "517656ddfd0a502e0294104fe71d34a05302b61498a49da054a68987fa455812", + "517656ddfd0a502e0294104fe71d34a05302b61498a49da054a68987fa455812" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "What is the weather like in Lisbon today? Answer as JSON.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "json_schema": { + "name": "weather", + "schema": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + } + }, + "type": "json_schema" + }, + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "361edec20fd0e7e737574dfaeec3d290f52a41f6b390a4a05fd488a0be413bf4", + "361edec20fd0e7e737574dfaeec3d290f52a41f6b390a4a05fd488a0be413bf4", + "361edec20fd0e7e737574dfaeec3d290f52a41f6b390a4a05fd488a0be413bf4" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "What is the weather like in Lisbon today? Answer as JSON.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "json_schema": { + "name": "weather", + "schema": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + } + }, + "type": "json_schema" + }, + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "a829538ceb2ceb71462719b460824cfc6abc31b4bbf089ac80266aad3998be70", + "a829538ceb2ceb71462719b460824cfc6abc31b4bbf089ac80266aad3998be70", + "a829538ceb2ceb71462719b460824cfc6abc31b4bbf089ac80266aad3998be70" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "What is the weather like in Lisbon today? Answer as JSON.", + "role": "user" + } + ], + "model": "lfm3-check", + "response_format": { + "json_schema": { + "name": "weather", + "schema": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + } + }, + "type": "json_schema" + }, + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "e334d523adbfbe07941af4f2efd60ba0955b74aa7ef7ff0d955dbaef5b2d684f", + "e334d523adbfbe07941af4f2efd60ba0955b74aa7ef7ff0d955dbaef5b2d684f", + "e334d523adbfbe07941af4f2efd60ba0955b74aa7ef7ff0d955dbaef5b2d684f" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_json": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + }, + "max_tokens": 400, + "messages": [ + { + "content": "Make up the weather in Oslo.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "ec0a75ef9736b0eaa4ddd2bb5ed64eb32080b2ecbcd9dfafa72ad32675eb0784", + "ec0a75ef9736b0eaa4ddd2bb5ed64eb32080b2ecbcd9dfafa72ad32675eb0784", + "ec0a75ef9736b0eaa4ddd2bb5ed64eb32080b2ecbcd9dfafa72ad32675eb0784" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_json": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Make up the weather in Oslo.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "342863a4b621d4d5c3b51d20415c53867cb35165f8fb08fd5d03e53d5661486f", + "342863a4b621d4d5c3b51d20415c53867cb35165f8fb08fd5d03e53d5661486f", + "342863a4b621d4d5c3b51d20415c53867cb35165f8fb08fd5d03e53d5661486f" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_json": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + }, + "max_tokens": 400, + "messages": [ + { + "content": "Make up the weather in Oslo.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "ffd631b0cc4d82afddc08ebb896fb32419da3f546dda4854762093a24b077f37", + "ffd631b0cc4d82afddc08ebb896fb32419da3f546dda4854762093a24b077f37", + "ffd631b0cc4d82afddc08ebb896fb32419da3f546dda4854762093a24b077f37" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_json": { + "properties": { + "city": { + "type": "string" + }, + "conditions": { + "enum": [ + "sunny", + "cloudy", + "rain", + "snow" + ] + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 3, + "type": "array" + }, + "temperature_c": { + "type": "integer" + } + }, + "required": [ + "city", + "temperature_c", + "conditions" + ], + "type": "object" + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Make up the weather in Oslo.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "858bc1207e3f40bac2597de755d39a6f66f82378691c31a78c565d7c3e30931a", + "858bc1207e3f40bac2597de755d39a6f66f82378691c31a78c565d7c3e30931a", + "858bc1207e3f40bac2597de755d39a6f66f82378691c31a78c565d7c3e30931a" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_regex": "\\d{4}-\\d{2}-\\d{2}", + "max_tokens": 400, + "messages": [ + { + "content": "When did the first person walk on the moon? Give the date.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "5ad44b55a70b073ffc05cd0d5da67f2d8d7ec609ce3ed820534ed69697ddbe76", + "5ad44b55a70b073ffc05cd0d5da67f2d8d7ec609ce3ed820534ed69697ddbe76", + "5ad44b55a70b073ffc05cd0d5da67f2d8d7ec609ce3ed820534ed69697ddbe76" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_regex": "\\d{4}-\\d{2}-\\d{2}", + "max_tokens": 1200, + "messages": [ + { + "content": "When did the first person walk on the moon? Give the date.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "508f46ee87de098b28863ee385c865240bfc57c9b1b794fa81390c0958659825", + "508f46ee87de098b28863ee385c865240bfc57c9b1b794fa81390c0958659825", + "508f46ee87de098b28863ee385c865240bfc57c9b1b794fa81390c0958659825" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_regex": "\\d{4}-\\d{2}-\\d{2}", + "max_tokens": 400, + "messages": [ + { + "content": "When did the first person walk on the moon? Give the date.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "466faf974ed2633c54f51bf9055f8c7f39c9e6a3fc9f759be47d5fe6f2521291", + "466faf974ed2633c54f51bf9055f8c7f39c9e6a3fc9f759be47d5fe6f2521291", + "466faf974ed2633c54f51bf9055f8c7f39c9e6a3fc9f759be47d5fe6f2521291" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_regex": "\\d{4}-\\d{2}-\\d{2}", + "max_tokens": 1200, + "messages": [ + { + "content": "When did the first person walk on the moon? Give the date.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "028e6b85824bac4cd03f1572b22f667f02bd10ff94be88cbf9b6cfceadcc9892", + "028e6b85824bac4cd03f1572b22f667f02bd10ff94be88cbf9b6cfceadcc9892", + "028e6b85824bac4cd03f1572b22f667f02bd10ff94be88cbf9b6cfceadcc9892" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_choice": [ + "positive", + "negative", + "neutral" + ], + "max_tokens": 400, + "messages": [ + { + "content": "Classify the sentiment: 'I loved every minute of this film.'", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d", + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d", + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_choice": [ + "positive", + "negative", + "neutral" + ], + "max_tokens": 1200, + "messages": [ + { + "content": "Classify the sentiment: 'I loved every minute of this film.'", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "896312515f03752a513f67ba40cb98b81b6212f4141dc783d93fcda3eada7b11", + "896312515f03752a513f67ba40cb98b81b6212f4141dc783d93fcda3eada7b11", + "896312515f03752a513f67ba40cb98b81b6212f4141dc783d93fcda3eada7b11" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_choice": [ + "positive", + "negative", + "neutral" + ], + "max_tokens": 400, + "messages": [ + { + "content": "Classify the sentiment: 'I loved every minute of this film.'", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d", + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d", + "28abb3eca0a660a77c572da20a61603b9ee466a3809c4bb98fcdf33912b09b3d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_choice": [ + "positive", + "negative", + "neutral" + ], + "max_tokens": 1200, + "messages": [ + { + "content": "Classify the sentiment: 'I loved every minute of this film.'", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "3e09b96b2542053ab74ead9d3b377a78936bf8f2a4775c5f2a61f31076f6bf04", + "3e09b96b2542053ab74ead9d3b377a78936bf8f2a4775c5f2a61f31076f6bf04", + "3e09b96b2542053ab74ead9d3b377a78936bf8f2a4775c5f2a61f31076f6bf04" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_grammar": "root ::= \"Answer: \" (\"yes\" | \"no\") \", because \" [a-z ]{3,40} \".\"", + "max_tokens": 400, + "messages": [ + { + "content": "Is the sky blue on a clear day?", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "1c4b25c4b3154630f336ea4e5c3f8029fab55a0a171c7b53a70ddfb5c9f5ab04", + "1c4b25c4b3154630f336ea4e5c3f8029fab55a0a171c7b53a70ddfb5c9f5ab04", + "1c4b25c4b3154630f336ea4e5c3f8029fab55a0a171c7b53a70ddfb5c9f5ab04" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_grammar": "root ::= \"Answer: \" (\"yes\" | \"no\") \", because \" [a-z ]{3,40} \".\"", + "max_tokens": 1200, + "messages": [ + { + "content": "Is the sky blue on a clear day?", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "352433279b31016172860966480593c25abd4083021e0ad2b2f4510a69e3f837", + "352433279b31016172860966480593c25abd4083021e0ad2b2f4510a69e3f837", + "352433279b31016172860966480593c25abd4083021e0ad2b2f4510a69e3f837" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "guided_grammar": "root ::= \"Answer: \" (\"yes\" | \"no\") \", because \" [a-z ]{3,40} \".\"", + "max_tokens": 400, + "messages": [ + { + "content": "Is the sky blue on a clear day?", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "ba64d4389e5d82b0a3e4119e04d1d99432567b6582c8e23e5db3a64ed5761415", + "ba64d4389e5d82b0a3e4119e04d1d99432567b6582c8e23e5db3a64ed5761415", + "ba64d4389e5d82b0a3e4119e04d1d99432567b6582c8e23e5db3a64ed5761415" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "guided_grammar": "root ::= \"Answer: \" (\"yes\" | \"no\") \", because \" [a-z ]{3,40} \".\"", + "max_tokens": 1200, + "messages": [ + { + "content": "Is the sky blue on a clear day?", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "fbe0f857a5b704d9b64254e165b8384cbb7a16f2108e4b5e4d9bdfde8ea7ee8d", + "fbe0f857a5b704d9b64254e165b8384cbb7a16f2108e4b5e4d9bdfde8ea7ee8d", + "fbe0f857a5b704d9b64254e165b8384cbb7a16f2108e4b5e4d9bdfde8ea7ee8d" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Give today's date in ISO form.", + "role": "user" + } + ], + "model": "lfm3-check", + "structured_outputs": { + "regex": "\\d{4}-\\d{2}-\\d{2}" + }, + "temperature": 0 + }, + "count": 3, + "hashes": [ + "a4397e998056b0d725b6151df114c2a4c916e9efce98a20f11ad6666c272bb39", + "a4397e998056b0d725b6151df114c2a4c916e9efce98a20f11ad6666c272bb39", + "a4397e998056b0d725b6151df114c2a4c916e9efce98a20f11ad6666c272bb39" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Give today's date in ISO form.", + "role": "user" + } + ], + "model": "lfm3-check", + "structured_outputs": { + "regex": "\\d{4}-\\d{2}-\\d{2}" + }, + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "3daa2e15ab32f03e3ff68a8c4d72b7fdc7662fb4eca448d9960644cac8251c1e", + "3daa2e15ab32f03e3ff68a8c4d72b7fdc7662fb4eca448d9960644cac8251c1e", + "3daa2e15ab32f03e3ff68a8c4d72b7fdc7662fb4eca448d9960644cac8251c1e" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Give today's date in ISO form.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "structured_outputs": { + "regex": "\\d{4}-\\d{2}-\\d{2}" + }, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "12c1bc9f4e0160d763f9ecedc28e39f81b66664c598cacdd60d3739678fefe81", + "12c1bc9f4e0160d763f9ecedc28e39f81b66664c598cacdd60d3739678fefe81", + "12c1bc9f4e0160d763f9ecedc28e39f81b66664c598cacdd60d3739678fefe81" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Give today's date in ISO form.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "structured_outputs": { + "regex": "\\d{4}-\\d{2}-\\d{2}" + }, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "9dd396aa81f2a111491d2ccf92bd6674845adabc14a3e295e4266ec50c63556f", + "9dd396aa81f2a111491d2ccf92bd6674845adabc14a3e295e4266ec50c63556f", + "9dd396aa81f2a111491d2ccf92bd6674845adabc14a3e295e4266ec50c63556f" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0 + }, + "count": 3, + "hashes": [ + "3594736c0c8b914daa5abd7a250ef25e43dd0818d86ab57e51c6acf8360c2144", + "3594736c0c8b914daa5abd7a250ef25e43dd0818d86ab57e51c6acf8360c2144", + "3594736c0c8b914daa5abd7a250ef25e43dd0818d86ab57e51c6acf8360c2144" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "temperature": 0, + "thinking_budget": 256 + }, + "count": 3, + "hashes": [ + "d84edd56121bbadfbd11fd5e3a5d6e048aae30d4de5152be87d8608f18011260", + "d84edd56121bbadfbd11fd5e3a5d6e048aae30d4de5152be87d8608f18011260", + "d84edd56121bbadfbd11fd5e3a5d6e048aae30d4de5152be87d8608f18011260" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": false + }, + "max_tokens": 400, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "3594736c0c8b914daa5abd7a250ef25e43dd0818d86ab57e51c6acf8360c2144", + "3594736c0c8b914daa5abd7a250ef25e43dd0818d86ab57e51c6acf8360c2144", + "3594736c0c8b914daa5abd7a250ef25e43dd0818d86ab57e51c6acf8360c2144" + ], + "verified": true + }, + { + "request": { + "chat_template_kwargs": { + "enable_thinking": true + }, + "max_tokens": 1200, + "messages": [ + { + "content": "Write two sentences about the sea.", + "role": "user" + } + ], + "model": "lfm3-check", + "seed": 1234, + "temperature": 0.8, + "thinking_budget": 256, + "top_p": 0.95 + }, + "count": 3, + "hashes": [ + "91a1e14a5b9fc3e27a8b4821548c7203af467f0ca1fdea0b4ba3156a2dd27a49", + "91a1e14a5b9fc3e27a8b4821548c7203af467f0ca1fdea0b4ba3156a2dd27a49", + "91a1e14a5b9fc3e27a8b4821548c7203af467f0ca1fdea0b4ba3156a2dd27a49" + ], + "verified": true + } + ] + } + ] +} diff --git a/docs/receipts/lfm3/text-contract-nf2.json b/docs/receipts/lfm3/text-contract-nf2.json new file mode 100644 index 0000000000..a43401a49d --- /dev/null +++ b/docs/receipts/lfm3/text-contract-nf2.json @@ -0,0 +1,10 @@ +{ + "scope": "Extracted text normal contract build and host verification after compensated-cancellation correction; no GPU run is asserted for this executable.", + "source_commit": "c53423d0edfe11936ce77825a72ba1d45b85c4f2", + "command": "zig build test-kernel-contract tf-kernel-contract -Dcpu=apple_m1 --summary all", + "summary": "Build Summary: 61/61 steps succeeded; 121/121 tests passed", + "binary_sha256": "8adc967eaaa618f957c26f7ef2e8abd6dfe0edeacf53d2f33ce17035b37cf5b7", + "log_sha256": "1a103b6409def0b10b99e6e1b9fb737f9c613cb0e57de8becf5258141a9c011b", + "host": "Apple M4 Pro, macOS 26.5.1", + "compiler": "Zig 0.17.0" +} diff --git a/docs/receipts/lfm3/text-foundation.json b/docs/receipts/lfm3/text-foundation.json new file mode 100644 index 0000000000..e2715ab3a7 --- /dev/null +++ b/docs/receipts/lfm3/text-foundation.json @@ -0,0 +1,2266 @@ +{ + "scope": "Actual text-only extraction; binned projection remains disabled by default", + "binary_sha256": "19da51a24a28b0c1c7c2dac7a77a3badce2789487ba1f6c087baa689bc15ffa2", + "hardware": "Apple M4 Pro, 48 GB, macOS 26.5.1", + "build_summary": "243/243 steps succeeded; 1/1 newly executed tests passed, other tests cached", + "python_tests": 35, + "served": [ + { + "bits": 4, + "evidence": { + "model": "LFM3-1.7B-MLX-4bit", + "binary_sha256": "19da51a24a28b0c1c7c2dac7a77a3badce2789487ba1f6c087baa689bc15ffa2", + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "45a6b44159ba", + "token_sha256": "45a6b44159ba07186369252ec789bae6d416aa96a3f9334bff78fad99c76cf39", + "output_sha256": "45845dddc56ef15fe86293099981fedd8fea0f9b18a3c6e190fd076d6f0ecb8f" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "c060bd0bcf5a", + "token_sha256": "c060bd0bcf5a99accef7b621eb113602aec54c723ad536b39cc59be966c9da0f", + "output_sha256": "ffabe976d8bfaf15e329d26965b3b6af57dc38ba159610d99d2b984be4405677" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "6936d334c5fa", + "token_sha256": "6936d334c5fa71b46f1463f0a41a1bec9c2191a8b04427ac3973cc970decd511", + "output_sha256": "5acb1b52c5f3ada89db8d0b9daac2ab1b42771116f19bac7e541c90ae2d6f053" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "b9e29e56abc8", + "token_sha256": "b9e29e56abc8ceddf1718ed57246cf830bfad51ae7467cde34099680227c0246", + "output_sha256": "ea6098063ae30786a2c14440104d62903854ffb2544ad22a5977b7be395bfbb3" + } + } + ], + "speed_qualified": false, + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6 + } + }, + { + "bits": 6, + "evidence": { + "model": "LFM3-1.7B-MLX-6bit", + "binary_sha256": "19da51a24a28b0c1c7c2dac7a77a3badce2789487ba1f6c087baa689bc15ffa2", + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "39034fe0ba11", + "token_sha256": "39034fe0ba116d398790a2d6f9f27897a9033aca2899d2e7243573269cce0bcd", + "output_sha256": "00441f346388cb9e3da29d4470b37667d77b431854660bd3baef2d0d70ab98b3" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "85f0d1e2d9c5", + "token_sha256": "85f0d1e2d9c58a1f9c8470e50653cdab415feafcae83fac7a0aea691d9fe80dc", + "output_sha256": "200dd537754951a6c8ae4fc4d3419f8dda3f67e88ccc4766f2bdcaa260652966" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0ba3f45e84a2", + "token_sha256": "0ba3f45e84a2d008ad278a00a0d0bb92e86ab473f1591dd7f0e0b9fc7e9097fd", + "output_sha256": "e8f89e14d184deba4de23ecb09ae19ff858a341ae382f77e95af132c252897c0" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "47d528141151", + "token_sha256": "47d5281411514b32fd43b317870df757e61e0fb4b0956373c2426391156d8c7d", + "output_sha256": "5f26af10f7388c9cecdd8db5d6e063bde4c1e779f7fd2b028902b9745233476b" + } + } + ], + "speed_qualified": false, + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6 + } + }, + { + "bits": 8, + "evidence": { + "model": "LFM3-1.7B-MLX-8bit", + "binary_sha256": "19da51a24a28b0c1c7c2dac7a77a3badce2789487ba1f6c087baa689bc15ffa2", + "tests": [ + { + "name": "code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "long_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + }, + { + "name": "sampled_chat/fresh_plain", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + }, + { + "name": "long_prefix/length_over_256", + "status": "passed", + "evidence": 910 + }, + { + "name": "code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "long_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "long_prefix/warm_reuses_and_matches_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 910, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/no_stale_prefix_reuse", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "edited_prefix/matches_cache_off", + "status": "passed", + "evidence": { + "prompt_tokens": 904, + "cached_tokens": 896, + "tokens": 24, + "cache_state": "reused", + "complete": true, + "token_sha": "aa7eae3dfd00", + "token_sha256": "aa7eae3dfd0032e7c2e2242fb31bb477741ec3b1431b9eb2bcf937254f60e351", + "output_sha256": "997ba91a7473fbabe7bef7f814cd62ffcdb9a0ee609e152cc3eb92f06fce589e" + } + }, + { + "name": "sampled_code/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + }, + { + "name": "sampled_code/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + }, + { + "name": "sampled_chat/plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + }, + { + "name": "sampled_chat/drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + }, + { + "name": "code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + ] + }, + { + "name": "chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + ] + }, + { + "name": "sampled_code/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + ] + }, + { + "name": "sampled_chat/concurrent_1_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + ] + }, + { + "name": "code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + ] + }, + { + "name": "chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + ] + }, + { + "name": "sampled_code/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + ] + }, + { + "name": "sampled_chat/concurrent_2_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + ] + }, + { + "name": "code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + ] + }, + { + "name": "chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + ] + }, + { + "name": "sampled_code/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + }, + { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "bab9f7949a4f", + "token_sha256": "bab9f7949a4f9172787d3085cdf9fbcc97ab3a726ea003f1424f004b6cb9accc", + "output_sha256": "ac481aa6abb18323cebe0a711f09d811d9f042d378e34a64baa00732cf950e4d" + } + ] + }, + { + "name": "sampled_chat/concurrent_4_equals_solo", + "status": "passed", + "evidence": [ + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + }, + { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "9a2ab2f611f2", + "token_sha256": "9a2ab2f611f250fbf53b1a6214d58d958399ee89ba4a0b1b9dac19e2a1023b4b", + "output_sha256": "8e775bc08453e668e686def6917f8116879362eb0d283f860eb1b41ab2e03093" + } + ] + }, + { + "name": "code/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 0, + "server_disconnections_after": 1, + "server_confirmed_cancel": true + } + }, + { + "name": "code/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "code/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 12, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "fb5e9db70542", + "token_sha256": "fb5e9db70542b902465657f3a2924dbf7dfb1a08e9cd3e039b6f6932c5bf9f11", + "output_sha256": "e45552fd633a22c288d8bb067fa82c891e3ca8cb2df3339558832c270178612f" + } + }, + { + "name": "chat/cancel_server_confirmed", + "status": "passed", + "evidence": { + "requested_tokens": 512, + "closed_before_done": true, + "server_disconnections_before": 1, + "server_disconnections_after": 2, + "server_confirmed_cancel": true + } + }, + { + "name": "chat/cancel_next_plain_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + }, + { + "name": "chat/cancel_next_drafted_equals_fresh", + "status": "passed", + "evidence": { + "prompt_tokens": 21, + "cached_tokens": 0, + "tokens": 24, + "cache_state": "cold", + "complete": true, + "token_sha": "0a26c7a004ed", + "token_sha256": "0a26c7a004edc59bc3a34b54d206656a5974430a51ff9c9691312fb8e2182ac2", + "output_sha256": "02e8cfe0c72109ca7a058c26aa569504d1995c20d3c49e7451c8ae963ae64f35" + } + } + ], + "speed_qualified": false, + "counts": { + "passed": 40, + "failed": 0, + "skipped": 0 + }, + "cancellation_checks": 6 + } + } + ], + "vl_refusals": [ + { + "bits": 4, + "vl_refusal": true, + "exit_status": 1 + }, + { + "bits": 6, + "vl_refusal": true, + "exit_status": 1 + }, + { + "bits": 8, + "vl_refusal": true, + "exit_status": 1 + } + ], + "targeted_contract": { + "filter": "LFM3", + "summary": "SUMMARY 49 kernels: 49 pass gates 1-4, 0 fail, 0 known findings; 14 controls: 14 red as planted, 0 missed; 0 limit problems; gates 64304 ms, limits 4564 ms", + "scope": "Registered text-extraction LFM3 probes and limits; not a complete backend run" + } +} diff --git a/docs/receipts/lfm3/text-twin-regression.json b/docs/receipts/lfm3/text-twin-regression.json new file mode 100644 index 0000000000..6a6260f864 --- /dev/null +++ b/docs/receipts/lfm3/text-twin-regression.json @@ -0,0 +1,10 @@ +{ + "scope": "Host/native/developer-tool regression after shared opt-in projection twin. No device/model qualification of this rebuilt binary is asserted.", + "source_commit": "5965e23d8e322275b8c813308c9d7edf72f001aa", + "command": "zig build native dev-tools test test-golden -Dcpu=apple_m1 --summary all", + "summary": "Build Summary: 721/721 steps succeeded; 7233/7278 tests passed (45 skipped)", + "native_binary_sha256": "0027180c77ef97cc3095c7596bd44aa05e9d3fe8ebe87abceb20123f515b627d", + "compiler": "Zig 0.17.0", + "host": "Apple M4 Pro, macOS 26.5.1", + "log_sha256": "89ddf8140264ee52b53be73ee0731252077cccd2f93f9c25e4adbd7c1bc6ee19" +} diff --git a/docs/receipts/lfm3/tool-checks.json b/docs/receipts/lfm3/tool-checks.json new file mode 100644 index 0000000000..4dd647404e --- /dev/null +++ b/docs/receipts/lfm3/tool-checks.json @@ -0,0 +1,958 @@ +{ + "cells": [ + { + "platform": "M5 Ultra, macOS 27.0", + "checkpoint": "1.7B-4", + "native_binary_sha256": "251079309919aa9b21ca8cc6a89c58fc3fc9a9baf800fa5ce6a6af313a5b5f65", + "counts": { + "passed": 34, + "failed": 0 + }, + "checks": [ + { + "name": "required/0/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ] + } + }, + { + "name": "required/0/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "draft_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "required/0/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "named/0/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ] + } + }, + { + "name": "named/0/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "draft_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "named/0/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "none/0/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [] + } + }, + { + "name": "none/0/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "7843d72119d60114d3bfbeeba016494527c318e6b933128fa3a0d190ea191659", + "draft_hash": "7843d72119d60114d3bfbeeba016494527c318e6b933128fa3a0d190ea191659" + } + }, + { + "name": "none/0/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "required/0/concurrent_0_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "named/0/concurrent_1_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "none/0/concurrent_2_hash", + "passed": true, + "details": { + "http": 200, + "hash": "7843d72119d60114d3bfbeeba016494527c318e6b933128fa3a0d190ea191659" + } + }, + { + "name": "required/0/concurrent_3_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "required/0/streamed_calls_and_hash", + "passed": true, + "details": { + "complete": true, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ], + "tensorfold": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lfm3", + "tokens_per_second": 376.55113393239196, + "seconds": 0.058425, + "prefill_seconds": 0.03929575, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039586625, + "sampling": "greedy", + "drafts": true, + "token_sha": "bd55d22b78c1", + "token_sha256": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "token_ids": [ + 125019, + 207, + 125021, + 2762, + 703, + 125023, + 73, + 703, + 26, + 207, + 125024, + 207, + 125023, + 74, + 703, + 27, + 207, + 125024, + 207, + 125022, + 207, + 125020, + 124900 + ], + "min_rows": 1 + } + } + }, + { + "name": "roundtrip/0/False/valid", + "passed": true, + "details": { + "http": 200, + "answer": { + "result": 5 + } + } + }, + { + "name": "roundtrip/0/True/valid", + "passed": true, + "details": { + "http": 200, + "answer": { + "result": 5 + } + } + }, + { + "name": "roundtrip/0/draft_hash_equals_plain", + "passed": true, + "details": null + }, + { + "name": "required/0.7/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ] + } + }, + { + "name": "required/0.7/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "draft_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "required/0.7/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "named/0.7/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ] + } + }, + { + "name": "named/0.7/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "draft_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "named/0.7/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "none/0.7/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [] + } + }, + { + "name": "none/0.7/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "b42001becb1b1816f0be07fe107d33efd346f492f3e1f0b41fa9314b1fccea51", + "draft_hash": "b42001becb1b1816f0be07fe107d33efd346f492f3e1f0b41fa9314b1fccea51" + } + }, + { + "name": "none/0.7/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "required/0.7/concurrent_0_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "named/0.7/concurrent_1_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "none/0.7/concurrent_2_hash", + "passed": true, + "details": { + "http": 200, + "hash": "b42001becb1b1816f0be07fe107d33efd346f492f3e1f0b41fa9314b1fccea51" + } + }, + { + "name": "required/0.7/concurrent_3_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "required/0.7/streamed_calls_and_hash", + "passed": true, + "details": { + "complete": true, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ], + "tensorfold": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lfm3", + "tokens_per_second": 371.2996571520229, + "seconds": 0.059251334, + "prefill_seconds": 0.039364709, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039642166, + "sampling": "exact", + "drafts": true, + "token_sha": "bd55d22b78c1", + "token_sha256": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "token_ids": [ + 125019, + 207, + 125021, + 2762, + 703, + 125023, + 73, + 703, + 26, + 207, + 125024, + 207, + 125023, + 74, + 703, + 27, + 207, + 125024, + 207, + 125022, + 207, + 125020, + 124900 + ], + "min_rows": 1 + } + } + }, + { + "name": "roundtrip/0.7/False/valid", + "passed": true, + "details": { + "http": 200, + "answer": { + "result": 5 + } + } + }, + { + "name": "roundtrip/0.7/True/valid", + "passed": true, + "details": { + "http": 200, + "answer": { + "result": 5 + } + } + }, + { + "name": "roundtrip/0.7/draft_hash_equals_plain", + "passed": true, + "details": null + } + ] + }, + { + "platform": "M5 Ultra, macOS 27.0", + "checkpoint": "VL-2B-4", + "native_binary_sha256": "251079309919aa9b21ca8cc6a89c58fc3fc9a9baf800fa5ce6a6af313a5b5f65", + "counts": { + "passed": 46, + "failed": 0 + }, + "checks": [ + { + "name": "required/0/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ] + } + }, + { + "name": "required/0/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "draft_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "required/0/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "named/0/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ] + } + }, + { + "name": "named/0/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "draft_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "named/0/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "none/0/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [] + } + }, + { + "name": "none/0/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "7843d72119d60114d3bfbeeba016494527c318e6b933128fa3a0d190ea191659", + "draft_hash": "7843d72119d60114d3bfbeeba016494527c318e6b933128fa3a0d190ea191659" + } + }, + { + "name": "none/0/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "image_red/0/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "report_color", + "arguments": { + "color": "red" + } + } + ] + } + }, + { + "name": "image_red/0/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "488388eba1eefda92bdedab254fa2ce0aa7e3b56d2af834bdaa88e031def833a", + "draft_hash": "488388eba1eefda92bdedab254fa2ce0aa7e3b56d2af834bdaa88e031def833a" + } + }, + { + "name": "image_red/0/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "image_blue/0/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "report_color", + "arguments": { + "color": "blue" + } + } + ] + } + }, + { + "name": "image_blue/0/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "762466499cd3ee7e4587a4911cbd779266f5dd58ca1d744e4c4d3f9d42995b11", + "draft_hash": "762466499cd3ee7e4587a4911cbd779266f5dd58ca1d744e4c4d3f9d42995b11" + } + }, + { + "name": "image_blue/0/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "required/0/concurrent_0_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "named/0/concurrent_1_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "none/0/concurrent_2_hash", + "passed": true, + "details": { + "http": 200, + "hash": "7843d72119d60114d3bfbeeba016494527c318e6b933128fa3a0d190ea191659" + } + }, + { + "name": "image_red/0/concurrent_3_hash", + "passed": true, + "details": { + "http": 200, + "hash": "488388eba1eefda92bdedab254fa2ce0aa7e3b56d2af834bdaa88e031def833a" + } + }, + { + "name": "required/0/streamed_calls_and_hash", + "passed": true, + "details": { + "complete": true, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ], + "tensorfold": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lfm3", + "tokens_per_second": 354.15443760661657, + "seconds": 0.062119792, + "prefill_seconds": 0.039358959, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039628792, + "sampling": "greedy", + "drafts": true, + "token_sha": "bd55d22b78c1", + "token_sha256": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "token_ids": [ + 125019, + 207, + 125021, + 2762, + 703, + 125023, + 73, + 703, + 26, + 207, + 125024, + 207, + 125023, + 74, + 703, + 27, + 207, + 125024, + 207, + 125022, + 207, + 125020, + 124900 + ], + "min_rows": 1 + } + } + }, + { + "name": "roundtrip/0/False/valid", + "passed": true, + "details": { + "http": 200, + "answer": { + "result": 5 + } + } + }, + { + "name": "roundtrip/0/True/valid", + "passed": true, + "details": { + "http": 200, + "answer": { + "result": 5 + } + } + }, + { + "name": "roundtrip/0/draft_hash_equals_plain", + "passed": true, + "details": null + }, + { + "name": "required/0.7/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ] + } + }, + { + "name": "required/0.7/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "draft_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "required/0.7/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "named/0.7/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ] + } + }, + { + "name": "named/0.7/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "draft_hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "named/0.7/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "none/0.7/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [] + } + }, + { + "name": "none/0.7/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "7843d72119d60114d3bfbeeba016494527c318e6b933128fa3a0d190ea191659", + "draft_hash": "7843d72119d60114d3bfbeeba016494527c318e6b933128fa3a0d190ea191659" + } + }, + { + "name": "none/0.7/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "image_red/0.7/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "report_color", + "arguments": { + "color": "red" + } + } + ] + } + }, + { + "name": "image_red/0.7/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "488388eba1eefda92bdedab254fa2ce0aa7e3b56d2af834bdaa88e031def833a", + "draft_hash": "488388eba1eefda92bdedab254fa2ce0aa7e3b56d2af834bdaa88e031def833a" + } + }, + { + "name": "image_red/0.7/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "image_blue/0.7/valid_plain", + "passed": true, + "details": { + "http": 200, + "calls": [ + { + "name": "report_color", + "arguments": { + "color": "blue" + } + } + ] + } + }, + { + "name": "image_blue/0.7/draft_hash_equals_plain", + "passed": true, + "details": { + "http": 200, + "plain_hash": "762466499cd3ee7e4587a4911cbd779266f5dd58ca1d744e4c4d3f9d42995b11", + "draft_hash": "762466499cd3ee7e4587a4911cbd779266f5dd58ca1d744e4c4d3f9d42995b11" + } + }, + { + "name": "image_blue/0.7/valid_drafted", + "passed": true, + "details": { + "http": 200 + } + }, + { + "name": "required/0.7/concurrent_0_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "named/0.7/concurrent_1_hash", + "passed": true, + "details": { + "http": 200, + "hash": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9" + } + }, + { + "name": "none/0.7/concurrent_2_hash", + "passed": true, + "details": { + "http": 200, + "hash": "7843d72119d60114d3bfbeeba016494527c318e6b933128fa3a0d190ea191659" + } + }, + { + "name": "image_red/0.7/concurrent_3_hash", + "passed": true, + "details": { + "http": 200, + "hash": "488388eba1eefda92bdedab254fa2ce0aa7e3b56d2af834bdaa88e031def833a" + } + }, + { + "name": "required/0.7/streamed_calls_and_hash", + "passed": true, + "details": { + "complete": true, + "calls": [ + { + "name": "add", + "arguments": { + "a": 2, + "b": 3 + } + } + ], + "tensorfold": { + "enable_thinking": false, + "reasoning_effort": "none", + "engine": "lfm3", + "tokens_per_second": 348.1274304635238, + "seconds": 0.06319525, + "prefill_seconds": 0.03948375, + "prefill_widths": [], + "prefill_raised": [], + "time_to_first_token": 0.039755708, + "sampling": "exact", + "drafts": true, + "token_sha": "bd55d22b78c1", + "token_sha256": "bd55d22b78c1c3967eb7220d15f52c4a2715c65fd978c612caa822472d8528e9", + "token_ids": [ + 125019, + 207, + 125021, + 2762, + 703, + 125023, + 73, + 703, + 26, + 207, + 125024, + 207, + 125023, + 74, + 703, + 27, + 207, + 125024, + 207, + 125022, + 207, + 125020, + 124900 + ], + "min_rows": 1 + } + } + }, + { + "name": "roundtrip/0.7/False/valid", + "passed": true, + "details": { + "http": 200, + "answer": { + "result": 5 + } + } + }, + { + "name": "roundtrip/0.7/True/valid", + "passed": true, + "details": { + "http": 200, + "answer": { + "result": 5 + } + } + }, + { + "name": "roundtrip/0.7/draft_hash_equals_plain", + "passed": true, + "details": null + } + ] + } + ] +} diff --git a/docs/receipts/lfm3/vl-twin-regression.json b/docs/receipts/lfm3/vl-twin-regression.json new file mode 100644 index 0000000000..886766e7e6 --- /dev/null +++ b/docs/receipts/lfm3/vl-twin-regression.json @@ -0,0 +1,11 @@ +{ + "scope": "Canonical default-OFF production source rebuild, including trace-only vision snapshots. No device/model pass is asserted for this rebuilt binary.", + "source_commit": "6101b94bf6e8e8e29a6cde588f88b7b5020d5e8c", + "command": "zig build native dev-tools test test-golden -Dcpu=apple_m1 --summary all", + "summary": "Build Summary: 726/726 steps succeeded; 982/1000 tests passed (18 skipped)", + "cached_steps": "Other steps reused cached checks; counts are those newly reported by Zig.", + "native_binary_sha256": "44220f39df174b2364172eac4b8705dbd5f5f47f3175c7ed300a6d47ae3bbcbc", + "log_sha256": "fe4dd4500935a9a64ad0af0e85e090a55dca18e8127889fe9b410617f146f156", + "host": "Apple M4 Pro, macOS 26.5.1", + "compiler": "Zig 0.17.0" +} diff --git a/docs/receipts/lfm3/vl4-code265-rounded-graph.json b/docs/receipts/lfm3/vl4-code265-rounded-graph.json new file mode 100644 index 0000000000..f44895a46f --- /dev/null +++ b/docs/receipts/lfm3/vl4-code265-rounded-graph.json @@ -0,0 +1,383 @@ +{ + "scope": "VL4 four-bit causal code prefix of 265 tokens: independent CPU prescribed-rounding graph with production-options Metal elementary evaluations. Only vocabulary row 264 is compared.", + "runtime_change": false, + "native_activation_injection": false, + "checkpoint": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "revision": "c823fade98e35a3d6942ad4d38b1ea15f675cd12" + }, + "prefix_rows": 265, + "final_row": 264, + "token_ids_sha256": "58ee46fd76e51555ec0fad27675aca85def7ea6406110f7aa9530a11eb94ee4d", + "token_ids": [ + 22497, + 267, + 21516, + 1446, + 395, + 16033, + 273, + 1046, + 38076, + 15493, + 2129, + 8864, + 3520, + 5635, + 22, + 42847, + 851, + 893, + 309, + 2993, + 16318, + 63908, + 3162, + 33237, + 116766, + 92377, + 20, + 1583, + 4673, + 360, + 1224, + 550, + 34289, + 360, + 723, + 550, + 495, + 550, + 229, + 24, + 207, + 22497, + 267, + 21516, + 1446, + 395, + 16033, + 273, + 1046, + 38076, + 15493, + 2129, + 8864, + 3520, + 5635, + 22, + 42847, + 851, + 893, + 309, + 2993, + 16318, + 63908, + 3162, + 33237, + 116766, + 92377, + 20, + 1583, + 4673, + 360, + 1224, + 550, + 34289, + 360, + 723, + 550, + 495, + 550, + 229, + 24, + 207, + 22497, + 267, + 21516, + 1446, + 395, + 16033, + 273, + 1046, + 38076, + 15493, + 2129, + 8864, + 3520, + 5635, + 22, + 42847, + 851, + 893, + 309, + 2993, + 16318, + 63908, + 3162, + 33237, + 116766, + 92377, + 20, + 1583, + 4673, + 360, + 1224, + 550, + 34289, + 360, + 723, + 550, + 495, + 550, + 229, + 24, + 207, + 22497, + 267, + 21516, + 1446, + 395, + 16033, + 273, + 1046, + 38076, + 15493, + 2129, + 8864, + 3520, + 5635, + 22, + 42847, + 851, + 893, + 309, + 2993, + 16318, + 63908, + 3162, + 33237, + 116766, + 92377, + 20, + 1583, + 4673, + 360, + 1224, + 550, + 34289, + 360, + 723, + 550, + 495, + 550, + 229, + 24, + 207, + 22497, + 267, + 21516, + 1446, + 395, + 16033, + 273, + 1046, + 38076, + 15493, + 2129, + 8864, + 3520, + 5635, + 22, + 42847, + 851, + 893, + 309, + 2993, + 16318, + 63908, + 3162, + 33237, + 116766, + 92377, + 20, + 1583, + 4673, + 360, + 1224, + 550, + 34289, + 360, + 723, + 550, + 495, + 550, + 229, + 24, + 207, + 22497, + 267, + 21516, + 1446, + 395, + 16033, + 273, + 1046, + 38076, + 15493, + 2129, + 8864, + 3520, + 5635, + 22, + 42847, + 851, + 893, + 309, + 2993, + 16318, + 63908, + 3162, + 33237, + 116766, + 92377, + 20, + 1583, + 4673, + 360, + 1224, + 550, + 34289, + 360, + 723, + 550, + 495, + 550, + 229, + 24, + 207, + 22497, + 267, + 21516, + 1446, + 395, + 16033, + 273, + 1046, + 38076, + 15493, + 2129, + 8864, + 3520, + 5635, + 22, + 42847, + 851, + 893, + 309 + ], + "measured_identity": { + "final_logits_exact": 128000, + "final_logits_total": 128000, + "native_top": 207, + "rounded_reference_top": 207, + "relative_RMS_vs_native": 0.0, + "compared_stage_count": 0, + "native_stage_trace_available": false + }, + "ideal_FP64_difference": { + "retained": true, + "contenders": { + "207": { + "native": 14.625, + "ideal_FP64": 14.4375, + "projection_only_FP32": 14.625, + "CPU_full_named": 14.625 + }, + "2993": { + "native": 14.125, + "ideal_FP64": 14.8125, + "projection_only_FP32": 14.0625, + "CPU_full_named": 14.125 + }, + "33237": { + "native": 14.375, + "ideal_FP64": 14.25, + "projection_only_FP32": 14.375, + "CPU_full_named": 14.375 + } + }, + "ideal_top": 2993, + "claim": "Reproducing the named rounding sites does not turn disagreement with ideal FP64 into an ideal-FP64 pass." + }, + "elementary_evaluations": { + "batches": 762, + "scalar_values": 74442293, + "operations": { + "rsqrt": 119, + "exp": 577, + "exp2": 22, + "cos": 22, + "sin": 22 + }, + "ordered_chunk_groups": 24, + "max_payload_bytes": 1048448, + "chunking": "Pointwise arrays are split into ordered chunks of at most 262112 FP32 values. Original array input/output hashes are reconstructed and checked.", + "inputs": "Independently computed CPU graph intermediates from checkpoint weights and token IDs; native artifacts are used only for comparison.", + "functions": { + "exp": "metal::precise::exp", + "exp2": "metal::exp2", + "cos": "metal::precise::cos", + "sin": "metal::precise::sin", + "rsqrt": "metal::precise::rsqrt" + }, + "CompileOptions": { + "math_mode": "safe", + "math_functions": "fast", + "language_code": 262144 + }, + "raw_calls_manifest_sha256": "0ed030f0f3e284718e9a3e47c56cd2b7dd63399cca7bf6c03714d4afe10453d7", + "raw_groups_manifest_sha256": "8d1939bd8bfb4a973defcc9824f4f99f32f1ba34e4c9eea9b5204c31ef3c0f15" + }, + "source_sha256": { + "fp64_reference.py": "0fb1128ade095db87b6126792ab14930792ab1f7b41b998570356bc586f16b2e", + "row_projection_reference.py": "d03f224a037a920ba66b9828059d1489e48729cd168077e76789d0ec5c199c0c", + "elementary_reference.py": "3e4db276278867b7683536044ed7bcbe4c7643867d722fdda5b8ad895b3963e7", + "text_rounding_reference.py": "52665eef48570d45a2554861fe8f22b31874f8f464366adf882f40c3d311db27", + "text_attention_rounding.py": "ae67f2b2c26f89f75b1f36822a4b30a9f710df2b42f9f7ec42679373028b89e8" + }, + "reference_C": { + "sha256": "ff336f75fcf0a4f6c0ac92ed627bc1a6abb0b36b8f4cc95d876f2a8bb2f5cc42", + "ABI": "lfm3_row_replay(float*,uint8*,float*,float*,float*,int rows,int N,int K,int group,int bits); recorded binary hash proves artifact identity, not reproducible compiler provenance" + }, + "artifact_sha256": { + "VL-2B-4-code-513.native.bf16": "83013e7646760ead9206c4a53b7cac56a19d11035629c876f9ea6701dddedc08", + "VL-2B-4-code-513.bf16": "61e6cd3d56a41b198ee9877a9a477ec61310ea4826869eef057da42cd8549808", + "code-513.npy": "c932ae46fd83f6e834a4001bd27496af3ebf54ac841a694d03a64558ba0daacb", + "run.py": "7da87e4f18dc43a325a6d3dd20b283d8fb5751e468f95fea843b79751ad9fb0a", + "provider.py": "97aec6e37977bbef32c60e7f86253c29c7b8161159dd6c9aded657e4bd69ddd0", + "row264.bf16": "55ae2380c9a29c927f1ac61d411961770b3039493be450cf0b0e0d2d8cc3484e", + "row.so": "11ad0f66632bfcba548dbede9027ffdb112cd418ff2be80610af56a957424753" + }, + "executed_manifest_sha256": "4d6de7da49b13a1aa07914e8db0ef7555e5894012752c4756a2dc474b426a2d8", + "guarded_execution": { + "lock": "/tmp/tensorfold-model.lock", + "inode": 46835039, + "single_model_admission_passed": true, + "graph_child_exit": 0, + "all_primitive_children_exited": true, + "lock_released_after_completion": true, + "events_sha256": "47f3445ffc6708277117db23cee79a42b3b68111b061ba77dfa974f00d84ad81" + }, + "limits": [ + "Ordinary attention reference lacks selected integer-exponent range repair; observed equality validates only measured inputs.", + "Reference fma uses FP64 multiply/add then FP32 and is not universally correctly rounded fmaf.", + "Masked elementary evaluations may include NaN/-Inf discarded by reference; these are not asserted as native executed sites.", + "No native code-513 stage trace was retained; zero activation stages are compared.", + "Only the four-bit prefix of 265 tokens and final row 264 are covered. No image, 6/8-bit, performance or full-fidelity gate is claimed." + ], + "setup_history": "Two private diagnostic setup attempts stopped before a numerical result: the non-inclusive file-read limit and an incorrect diagnostic language-code assertion. Both were preserved; no runtime arithmetic changed.", + "fidelity_pass": null, + "accepted_threshold": null, + "original_graph_receipt_sha256": "4cd7c5b0e4ac96fc212e81d6b06c7ba2216bbd1ec4033373ea320a636e797dc5" +} diff --git a/docs/receipts/lfm3/vl4-ownvision41-rounded-graph.json b/docs/receipts/lfm3/vl4-ownvision41-rounded-graph.json new file mode 100644 index 0000000000..30cb22922f --- /dev/null +++ b/docs/receipts/lfm3/vl4-ownvision41-rounded-graph.json @@ -0,0 +1,194 @@ +{ + "scope": "Four-bit real-image own-vision replay: independently compute all 70 projected image rows, then decoder prefix41 and vocabulary heads34/40 only.", + "runtime_change": false, + "native_activation_injection": false, + "checkpoint": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "revision": "c823fade98e35a3d6942ad4d38b1ea15f675cd12" + }, + "image": { + "url": "https://placecats.com/300/200", + "jpeg_sha256": "d1b324f529a96d8d8a16d973d88e1e974905d9c2c0e3e8b21a50b1c0e3fe2379", + "canonical_png_sha256": "005500792ee8ea6f35ac369d900fc788cd77667f145e7100c2cff57fae5355d9", + "conversion": "macOS sips JPEG to PNG; canonical PNG is the exact native/reference input" + }, + "input_grid": [ + 14, + 20 + ], + "pixel_patches_shape": [ + 280, + 1536 + ], + "projected_image_rows_shape": [ + 70, + 2048 + ], + "decoder_image_start": 4, + "decoder_independent_image_rows_used": 37, + "decoder_prefix_rows": 41, + "token_ids": [ + 124899, + 5922, + 207, + 125009, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030, + 125030 + ], + "measured_identity": { + "independent_projected_BF16_words_equal": 143360, + "projected_BF16_words_total": 143360, + "heads": [ + { + "row": 34, + "native_equal_logits": 128000, + "values": 128000, + "native_top": 5594, + "independent_named_top": 5594, + "ideal_FP64_top": 2911, + "contenders": { + "2911": { + "native": 4.40625, + "independent_named": 4.40625, + "ideal_FP64": 4.5625 + }, + "5594": { + "native": 4.65625, + "independent_named": 4.65625, + "ideal_FP64": 4.53125 + } + } + }, + { + "row": 40, + "native_equal_logits": 128000, + "values": 128000, + "native_top": 4899, + "independent_named_top": 4899, + "ideal_FP64_top": 34541, + "contenders": { + "4899": { + "native": 7.375, + "independent_named": 7.375, + "ideal_FP64": 7.125 + }, + "34541": { + "native": 7.25, + "independent_named": 7.25, + "ideal_FP64": 7.1875 + } + } + } + ], + "compared_intermediate_stages": 0, + "native_stage_trace_available": false + }, + "independence": "Only checkpoint weights and frozen preprocessed image pixels enter VisionOracle. All70 projected rows are computed before selecting its first37 rows for the causal decoder prefix. Native image/logit artifacts are read only for comparison.", + "elementary_evaluations": { + "batches": 648, + "scalar_values": 33924773, + "operations": { + "rsqrt": 161, + "exp": 385, + "exp2": 34, + "cos": 34, + "sin": 34 + }, + "ordered_chunk_groups": 39, + "max_payload_bytes": 1048448, + "functions": { + "exp": "metal::precise::exp", + "exp2": "metal::exp2", + "cos": "metal::precise::cos", + "sin": "metal::precise::sin", + "rsqrt": "metal::precise::rsqrt" + }, + "CompileOptions": { + "math_mode": "safe", + "math_functions": "fast", + "language_code": 262144 + }, + "calls_manifest_sha256": "9b23c8b5717c5f4e607b3b2cc6d65ed4514e8c8405c8c69f38e1c15fe45cea59", + "groups_manifest_sha256": "2c02e290c38501ba102db8b629a3e331f62aaaae87d33c2b780a397ec95e323b" + }, + "source_sha256": { + "fp64_reference.py": "0fb1128ade095db87b6126792ab14930792ab1f7b41b998570356bc586f16b2e", + "row_projection_reference.py": "d03f224a037a920ba66b9828059d1489e48729cd168077e76789d0ec5c199c0c", + "elementary_reference.py": "3e4db276278867b7683536044ed7bcbe4c7643867d722fdda5b8ad895b3963e7", + "vision_fp64_reference.py": "742b8b4f44dde824c3a5c2d72a632401a0f9df7b7f23a8d3729d15f4f8c6a4d2", + "text_rounding_reference.py": "52665eef48570d45a2554861fe8f22b31874f8f464366adf882f40c3d311db27", + "vision_rounding_reference.py": "8b2f5679bbd3f05b3e86a5748c3803f744502cffd0159bee2a218da38bd6f1c4", + "text_attention_rounding.py": "ae67f2b2c26f89f75b1f36822a4b30a9f710df2b42f9f7ec42679373028b89e8" + }, + "artifact_sha256": { + "run.py": "866a4c925e7abe43188164cad2fa48c1648db1fca8a6a629f97cfbe4b4dfbbed", + "provider.py": "97aec6e37977bbef32c60e7f86253c29c7b8161159dd6c9aded657e4bd69ddd0", + "lfm3-real-cat-patches.npy": "f1c55a0dd28b91d9929f074995adadea79d7c0862c8cdcd0be6dbfda57383dc1", + "lfm3-real-cat-tokens.npy": "54c970db8c6ba82895cb2f18a48d397ea568ae885241129e9c5ee7dbbc91a24d", + "lfm3-real-cat-native.bf16": "ab2ac7db759e5c4a32fe7d60680ec47f7d1cbc5938fa7d8091bf4756bfeea4fc", + "lfm3-real-cat-native-text.bf16": "d2dc2c1014716cb9ead7543ee00562a1816bf50d4a18b927cd8d8151f0074d54", + "lfm3-real-cat-fp64-text.bf16": "c16af1c1ff0095b2ced285886f4c54e21c48f244cb74b4fa615c8954163e620f", + "row.so": "11ad0f66632bfcba548dbede9027ffdb112cd418ff2be80610af56a957424753", + "row_projection_replay.c": "ff336f75fcf0a4f6c0ac92ed627bc1a6abb0b36b8f4cc95d876f2a8bb2f5cc42" + }, + "independent_image70_sha256": "ab2ac7db759e5c4a32fe7d60680ec47f7d1cbc5938fa7d8091bf4756bfeea4fc", + "two_heads_sha256": "b412201478b2c3335691675e069eb86d92c4db98e901bd67b849e8f86ed4724a", + "executed_manifest_sha256": "bdbf40ef29b656ebe9a91ed282a38f4a76742146dbf326c26ea7542ca315c483", + "original_graph_receipt_sha256": "e23628491a8cbbea05f8746ba97030996d24020428bd328043b7f6acc2018dfc", + "guarded_execution": { + "lock": "/tmp/tensorfold-model.lock", + "inode": 46835039, + "single_model_admission_passed": true, + "graph_child_exit": 0, + "all_primitive_children_exited": true, + "lock_released_after_completion": true, + "events_sha256": "d9c624a97b385a7529f01e8a78c82893a53f6186ebf5758c9ad180e4679ba07c" + }, + "ideal_FP64_difference_retained": true, + "limits": [ + "No intermediate native stage comparison; stage hashes only.", + "Ordinary text attention reference lacks selected integer-exponent repair.", + "FP64 multiply/add then FP32 fma helper is not universally correctly rounded fmaf.", + "Observed graph equality only; ideal FP64 differences are retained without threshold change.", + "The selected decoder rows are inside the image-placeholder span, not generated suffix text.", + "No performance, 6/8-bit, all-position or final fidelity acceptance is claimed.", + "The dense C helper and row helper have pinned source/binary identity; historical binary compiler provenance is limited." + ], + "fidelity_pass": null, + "accepted_threshold": null +} diff --git a/docs/receipts/lfm3/vl4-prefix18-rounded-graph.json b/docs/receipts/lfm3/vl4-prefix18-rounded-graph.json new file mode 100644 index 0000000000..f4a254f121 --- /dev/null +++ b/docs/receipts/lfm3/vl4-prefix18-rounded-graph.json @@ -0,0 +1,2917 @@ +{ + "scope": "VL4 four-bit causal prefix18 independent CPU prescribed-rounding graph with production-options Metal elementary evaluations; finalvocabularyrow17 only", + "runtime_change": false, + "native_activation_injection": false, + "tokens": 18, + "final_row": 17, + "measured_identity": { + "matched_stages": 310, + "stage_BF16_bytes": 37349568, + "all_stages_exact": true, + "final_logits_exact": 128000, + "final_logits_total": 128000, + "native_top": 296, + "rounded_reference_top": 296, + "relative_RMS_vs_native": 0.0 + }, + "ideal_FP64_difference": { + "retained": true, + "contenders": { + "296": { + "native": 12.0625, + "ideal_FP64": 12.125, + "projection_only_FP32": 12.1875, + "CPU_full_named": 12.0625 + }, + "405": { + "native": 11.875, + "ideal_FP64": 12.25, + "projection_only_FP32": 12.125, + "CPU_full_named": 11.875 + } + }, + "claim": "This rounding reproduction does not turn ideal FP64 disagreement into an ideal-FP64 pass." + }, + "elementary_evaluations": { + "batches": 264, + "scalar_values": 3492778, + "operations": { + "rsqrt": 119, + "exp": 79, + "exp2": 22, + "cos": 22, + "sin": 22 + }, + "inputs": "Independently generated CPU graph intermediates from checkpoint and token IDs; native stages are comparison-only.", + "functions": { + "exp": "metal::precise::exp", + "exp2": "metal::exp2", + "cos": "metal::precise::cos", + "sin": "metal::precise::sin", + "rsqrt": "metal::precise::rsqrt" + }, + "CompileOptions": { + "math_mode": "safe", + "math_functions": "fast", + "language_code": 262144 + }, + "all_helper_exits_zero": true, + "raw_calls_manifest_sha256": "7653c5138260c0b5f37d8c6e0b1da6e03b86d810fe7dd4cdbdc92c0361bd8ee8" + }, + "stages": [ + { + "stage": "embedding", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_00_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_01_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_02_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_03_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_04_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_05_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_06_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_07_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_08_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_09_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_10_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_11_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_12_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_13_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_14_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_15_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_16_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_17_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_18_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_19_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_20_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_21_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_attn_q", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_attn_k", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_attn_v", + "rows": 18, + "width": 512, + "equal": 9216, + "values": 9216, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_attn_queries", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_attn_gate", + "rows": 18, + "width": 16, + "equal": 288, + "values": 288, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_attn_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_22_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_operator_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_conv_in", + "rows": 18, + "width": 6144, + "equal": 110592, + "values": 110592, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_conv_gated", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_mixer", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_post_operator", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_ffn_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_gate", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_up", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_act", + "rows": 18, + "width": 7680, + "equal": 138240, + "values": 138240, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_down", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "layer_23_output", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + }, + { + "stage": "final_norm", + "rows": 18, + "width": 2048, + "equal": 36864, + "values": 36864, + "row17_changed": 0, + "first_changed": null + } + ], + "source_sha256": { + "fp64_reference.py": "0fb1128ade095db87b6126792ab14930792ab1f7b41b998570356bc586f16b2e", + "row_projection_reference.py": "d03f224a037a920ba66b9828059d1489e48729cd168077e76789d0ec5c199c0c", + "elementary_reference.py": "3e4db276278867b7683536044ed7bcbe4c7643867d722fdda5b8ad895b3963e7", + "text_rounding_reference.py": "52665eef48570d45a2554861fe8f22b31874f8f464366adf882f40c3d311db27", + "text_attention_rounding.py": "ae67f2b2c26f89f75b1f36822a4b30a9f710df2b42f9f7ec42679373028b89e8" + }, + "reference_C": { + "sha256": "ff336f75fcf0a4f6c0ac92ed627bc1a6abb0b36b8f4cc95d876f2a8bb2f5cc42", + "ABI": "lfm3_row_replay(float*,uint8*,float*,float*,float*,int rows,int N,int K,int group,int bits); recorded binary hash proves artifact identity, not reproducible compiler provenance" + }, + "artifact_sha256": { + "VL-2B-4-reasoning-257.native.bf16": "118440ccd7fc07a502485a3266a2878ec815f316bcfa9dddd859e5d324fb8654", + "VL-2B-4-reasoning-257.bf16": "eaad83f4ec389f5c731c0d3422f2af47f218f8cdb220a6d0b752641c8f8f9bfe", + "lfm3-rounded-projection-reference.bf16": "b59c43083bfc6008233e7e249ced2f589ace92e1461e95c65300141aea14ab2b", + "reasoning-257.npy": "81449804fdd1a104d6dc9607c27d5cea49237064e50c80a765f68d8eba57f596", + "run.py": "adfd34f5ae830a4e06c1241b2f242da87bdb5d738f55fbfc98507ff8479a2c5e", + "provider.py": "e79e2f0ed4a881ca3cee683764a5eca3d9c4cc0c5aa9d27770bf69f41e922d38", + "row17.bf16": "acae5ff71fd78e34579440ad7609661ae366f423c54d3180b78bb7ff01086e35", + "row.so": "11ad0f66632bfcba548dbede9027ffdb112cd418ff2be80610af56a957424753" + }, + "graph_receipt_sha256": "6f493f08abf49aacbf3b8103d1dfb10ac40656f0c172c002a0f828881688db4c", + "guarded_execution": { + "lock": "/tmp/tensorfold-model.lock", + "inode": 46835039, + "single_model_admission_passed": true, + "graph_child_exit": 0, + "all_primitive_children_exited": true, + "lock_released_after_completion": true, + "events_sha256": "091742c894ffebc8d414eb56d18db1483f1c42bd1bf306523ab8126c6918aedd" + }, + "limits": [ + "Ordinary attention reference lacks selected integer-exponent range repair; observed equality validates only measured inputs.", + "Reference fma uses FP64 multiply/add then FP32 and is not universally correctly rounded fmaf.", + "Masked elementary evaluations may include NaN/-Inf discarded by reference; these are not asserted as native executed sites.", + "Only this four-bit prefix18 and finalrow17 are covered; no realimage, six/eight-bit, long-context, performance, or full-fidelity gate is claimed." + ], + "fidelity_pass": null, + "accepted_threshold": null, + "checkpoint": { + "repository": "LiquidAI/LFM3-VL-2B-MLX-4bit", + "revision": "c823fade98e35a3d6942ad4d38b1ea15f675cd12" + }, + "token_ids": [ + 41, + 9798, + 677, + 1703, + 50474, + 22, + 440, + 1008, + 10494, + 229, + 1325, + 5435, + 20, + 278, + 1840, + 10494, + 10625, + 405 + ], + "executed_manifest_sha256": "670b81847ba268197c5afaea9e5583a0b621bfbb77fa0c9806d9d28718977217" +} diff --git a/docs/recipes/lfm3-native.md b/docs/recipes/lfm3-native.md new file mode 100644 index 0000000000..859345f572 --- /dev/null +++ b/docs/recipes/lfm3-native.md @@ -0,0 +1,126 @@ +# LFM3 native Metal bring-up + +Tracking issue: [#1491](https://github.com/ashhart/TensorFold/issues/1491). + +The native Metal engine implements LFM3-1.7B text generation and LFM3-VL-2B image-to-text generation. +Both read the official MLX affine 4-, 6-, and 8-bit checkpoints without converting stored weights. +This is an experimental family port. Device and model checks cover an M4 Pro and an M5 Ultra; they do not qualify other Apple GPUs. + +## Weight licence + +The official text and VL MLX exports retain the **LFM Open License v1.0**, rather than TensorFold’s Apache-2.0 code licence. +See the [text weights licence](https://huggingface.co/LiquidAI/LFM3-1.7B-MLX-4bit/blob/main/LICENSE) and +[VL weights licence](https://huggingface.co/LiquidAI/LFM3-VL-2B-MLX-4bit/blob/main/LICENSE). +The checked Macs are an M4 Pro with 48 GB RAM and an M5 Ultra with 256 GiB RAM. + +## Run the models + +Build the native engine with the Zig version in `.zig-version` and Xcode's Metal toolchain: + +```sh +zig build native -Dcpu=apple_m1 +./zig-out/native/bin/tensorfold-native serve LiquidAI/LFM3-1.7B-MLX-4bit --context 8192 +./zig-out/native/bin/tensorfold-native serve LiquidAI/LFM3-VL-2B-MLX-4bit --context 8192 +``` + +Replace `4bit` with `6bit` or `8bit` for those exports. A complete local checkpoint folder also works. +The VL checkpoint loads its own image tower. Send images through OpenAI `image_url` content parts. +The default image budget is 256 tokens per request, with at most four images. +Video input is not implemented. JPEG and WebP decoding require the matching shared decoder libraries. + +The shared lane engine handles target verification, sampling and concurrent streams. +Convolution history and KV state are kept per stream; partial acceptance restores the accepted row's state. +Raw prompts of at least 256 tokens can build a repeat-cache entry: the first request records interest, +the second keeps a prefix state, and a later repeat resumes it. Image cache identity includes image content and grid. + +```mermaid +flowchart LR + I[Image pixels] --> V[Vision encoder and projector] + V --> E[Placeholder embedding rows] + T[Text tokens] --> E + E --> D[Shared LFM3 decoder] + D --> L[Existing lane engine] +``` + +## Inspect a checkpoint + +Use Python 3.11 or newer. The inspector uses the standard library. It resolves a full Hugging Face commit, +reads the config and index, and requests only the safetensors headers. It refuses a server that returns a +complete weight file instead of the requested byte range. + +```sh +python3 tools/lfm3/inspect_checkpoint.py LiquidAI/LFM3-1.7B-MLX-4bit --output text4.json +python3 tools/lfm3/inspect_checkpoint.py LiquidAI/LFM3-VL-2B-MLX-4bit --output vl4.json +``` + +Replace `4bit` with `6bit` or `8bit` to inspect those exports. Pass `--revision COMMIT` to repeat a pinned inspection. +Use `--local-dir MODEL_DIR` instead of a repository name to inspect files already on disk. + +## Check the native text layout + +Use the Zig version in `.zig-version`. + +```sh +zig build test-lfm3-host tf-lfm3-check +./zig-out/bin/tf-lfm3-check text4.json +./zig-out/bin/tf-lfm3-check vl4.json +python3 -m unittest discover -s tools/lfm3 -p 'test_*.py' -v +``` + +The native checker reads the saved metadata, validates shard header geometry, and checks the LFM3 text decoder's +required names, dtypes, shapes, and byte counts. It performs no projection, GPU allocation, or model inference. +For VL, it checks the text decoder only. A passing result does not validate vision weights or image processing. + +The config reader preserves full-attention versus sliding-attention layers, distinct rotary settings, sandwich norms, +and each module's affine width and group size. Unsupported geometry and conflicting quantization declarations fail. + +## Checkpoint details + +All six inspected MLX exports store the decoder under `language_model.model.*`, including the text-only model. +Each decoder has 594 tensors. Convolution kernels are BF16 with shape `[2048, 7, 1]`. + +Text 4/6/8-bit and VL 6/8-bit use affine groups of 64. VL 4-bit uses groups of 32 and an 8-bit embedding override. +The 138 additional VL vision/projector tensors are BF16 in all three exports. Global quantization metadata must not +cause these dense tensors to be treated as packed affine weights. + +The header inspections used these revisions: + +- Text 4-bit: `d0ca73dcfa70ce18733754a492cdc1fb0123262d`. +- Text 6-bit: `26be898c36b545d625d84ac0761c6c3eb664aef3`. +- Text 8-bit: `ee7667f3cfccb2a41725f91820531823fcabc6b7`. +- VL 4-bit: `c823fade98e35a3d6942ad4d38b1ea15f675cd12`. +- VL 6-bit: `6d71bfae121abedc75f6d531fc7f6ceb4f5e3501`. +- VL 8-bit: `83336d5be3874a2bdd3297bde77b2114dbd3a816`. + +## Qualification and remaining gate + +Detailed counts, hashes, numerical evidence and reproducible requests are in the [receipts](../receipts/lfm3/README.md). + +All six checkpoints passed 303 greedy/seeded-sampling, cache, concurrency and cancellation checks on one optimized M4 binary. +The final source build also passed 303/303 on an M5. Text4 and VL4 additionally passed real grammar, thinking, tool, +streaming, history and image-conditioned tool checks with the matching optional grammar library. +The 4-bit text long exactness suite passed all 73 cases through 8,193-token seams. +The final family contract passed 50 kernels and caught all 13 real controls, with zero limit problems. + +Independent projection audits covered 66 class/layout cells and 60,954,624 outputs with no violations of existing FP32/BF16 bounds. +The row-pair optimization preserves the shared arithmetic: 132 checkpoint calls and a full 8,193-token logit comparison were byte-identical. +On the M5, controlled 4-bit cold requests improved text engine prefill from 1.240 to 0.813 seconds at 2k and +73.402 to 61.633 seconds at 64k; VL decoder requests improved from 1.262 to 0.815 and 73.920 to 59.647 seconds. +These are engine prefill measurements; the receipt also retains client TTFT. All compared full output-token hashes matched. + +Native inference retains the checkpoint's BF16 activation boundaries and FP32 reductions. +Text trigonometric values round to BF16 before rotation; vision axial trigonometric values remain FP32. +The CPU FP64 oracle records numerical differences rather than inventing a tie policy. +MLX's vector/matrix quantized paths use different effective reconstruction, and bounded FP32 differences can propagate through history. +See the exact primitive and whole-model evidence before interpreting a next-token mismatch. + +```sh +zig build tf-lfm3-forward tf-lfm3-vision tf-lfm3-project tf-lfm3-profile tf-lfm3-exact -Dcpu=apple_m1 +./zig-out/bin/tf-lfm3-exact MODEL_DIR --long +python3 tools/lfm3/serve_check.py --binary ./zig-out/native/bin/tensorfold-native --model-dir MODEL_DIR --sampled --cancel --output served.json +``` + +Long-context model fidelity, maximum image budgets, 6/8-bit performance and other Apple GPU generations remain separate qualification work. +Structured output requires the native grammar library; logprobs and video are not advertised. +A complete earlier global M5 contract run found one Qwen4 failure that also reproduces on the unchanged pinned base. +It is not a final-source global pass. Agree model-fidelity and performance gates on #1491 before claiming release readiness. diff --git a/tools/lfm3/bench_decode.py b/tools/lfm3/bench_decode.py new file mode 100644 index 0000000000..95b0ba4ef3 --- /dev/null +++ b/tools/lfm3/bench_decode.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Run existing served benchmarks with public code/prose prompts long enough for 256-token comparisons.""" +import json +from pathlib import Path +import sys +import urllib.request + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import bench_concurrent +import bench_openai + +PROMPTS = [ + {'name': 'long-code-chat', 'kind': 'chat', 'prompt': + 'Write a complete Python implementation of a red-black tree with insertion, deletion, searching, ' + 'rotations, and extensive unit tests. Include detailed explanations after each method. ' + 'Continue until the implementation and every test are complete.'}, + {'name': 'long-prose-chat', 'kind': 'chat', 'prompt': + 'Write a detailed 2000-word tutorial explaining how matrix multiplication runs on a GPU, including ' + 'memory layout, tiling, shared memory, synchronization, numerical precision, and several worked ' + 'numerical examples. Explain every step thoroughly.'}, +] + + +def main(): + if len(sys.argv) < 2 or sys.argv[1] not in ('decode', 'concurrent'): + raise SystemExit('Choose decode or concurrent, then pass the existing benchmark client arguments') + mode = sys.argv.pop(1) + module = bench_openai if mode == 'decode' else bench_concurrent + module.PROMPTS = PROMPTS + original = urllib.request.Request + + def request(url, data=None, *args, **kwargs): + if data: + body = json.loads(data) + body.update(top_p=1, top_k=0, repetition_penalty=1, enable_thinking=False) + data = json.dumps(body).encode() + return original(url, data, *args, **kwargs) + + urllib.request.Request = request + try: + module.main() + finally: + urllib.request.Request = original + + +if __name__ == '__main__': + main() diff --git a/tools/lfm3/compare_logits.py b/tools/lfm3/compare_logits.py new file mode 100644 index 0000000000..5a981ef0e3 --- /dev/null +++ b/tools/lfm3/compare_logits.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Report every teacher-forced top-token difference without applying an acceptance tolerance.""" +import argparse +import hashlib +import json +from pathlib import Path +import numpy as np + + +def compare(native, reference, row_offset=0): + native, reference = np.asarray(native), np.asarray(reference) + if native.shape != reference.shape or native.ndim != 2 or native.shape[1] < 2 or not len(native): + raise ValueError('Expected equally shaped, nonempty row-by-vocabulary arrays') + if not np.isfinite(native).all() or not np.isfinite(reference).all(): + raise ValueError('Logits must be finite') + differences, worst = [], [] + sum_squared, reference_squared, maximum = 0.0, 0.0, 0.0 + column_squared = np.zeros(native.shape[1], dtype=np.float64) + column_maximum = np.zeros(native.shape[1], dtype=np.float64) + for index, (actual, expected) in enumerate(zip(native, reference)): + actual, expected = actual.astype(np.float64), expected.astype(np.float64) + error = actual - expected + square = float(error @ error) + denominator = float(expected @ expected) + column_squared += error * error + column_maximum = np.maximum(column_maximum, np.abs(error)) + sum_squared += square + reference_squared += denominator + maximum = max(maximum, float(np.abs(error).max())) + worst.append({'row': index + row_offset, 'rms_error': float(np.sqrt(square / len(error))), + 'maximum_absolute_error': float(np.abs(error).max())}) + a, b = int(actual.argmax()), int(expected.argmax()) + if a == b: + continue + aset = np.flatnonzero(actual == actual[a]).tolist() + bset = np.flatnonzero(expected == expected[b]).tolist() + atop = np.partition(actual, -2)[-2:] + btop = np.partition(expected, -2)[-2:] + differences.append({'row': index + row_offset, 'native_top': a, 'reference_top': b, + 'native_maximum_set': aset, 'reference_maximum_set': bset, + 'maximum_sets_overlap': bool(set(aset) & set(bset)), + 'native_margin': float(atop[-1] - atop[-2]), + 'reference_margin': float(btop[-1] - btop[-2]), + 'native_reference_contender_gap': float(actual[a] - actual[b]), + 'reference_native_contender_gap': float(expected[b] - expected[a])}) + columns = np.argsort(column_squared)[-20:][::-1] + worst_columns = [{'column': int(c), 'rms_error': float(np.sqrt(column_squared[c] / len(native))), + 'maximum_absolute_error': float(column_maximum[c])} for c in columns] + return {'rows': len(native), 'vocabulary': native.shape[1], 'top1_differences': len(differences), + 'disjoint_maximum_sets': sum(not item['maximum_sets_overlap'] for item in differences), + 'relative_rms': float(np.sqrt(sum_squared / reference_squared)) if reference_squared else None, + 'maximum_absolute_error': maximum, 'differences': differences, + 'worst_rows': sorted(worst, key=lambda item: item['rms_error'], reverse=True)[:20], + 'worst_columns': worst_columns, 'accepted_threshold': None, 'fidelity_pass': None} + + +def read_bf16(path, vocabulary): + if path.stat().st_size % 2: + raise ValueError('BF16 file contains an incomplete two-byte value') + values = np.fromfile(path, dtype=' len(tokens): + parser.error('Measured rows exceed teacher-forced fixture') + result = compare(native, reference, args.row_offset) + result.update({'phase': args.phase, + 'token_fixture_sha256': hashlib.sha256(args.tokens.read_bytes()).hexdigest(), + 'native_logits_sha256': hashlib.sha256(args.native.read_bytes()).hexdigest(), + 'reference_logits_sha256': hashlib.sha256(args.reference.read_bytes()).hexdigest(), + 'input_identity': 'Caller declares fixture; matching run commands establish execution identity'}) + args.output.write_text(json.dumps(result, indent=2, allow_nan=False) + '\n') + + +if __name__ == '__main__': + main() diff --git a/tools/lfm3/elementary_reference.py b/tools/lfm3/elementary_reference.py new file mode 100644 index 0000000000..6ba6d5848d --- /dev/null +++ b/tools/lfm3/elementary_reference.py @@ -0,0 +1,41 @@ +"""Shared reference-only FP32 elementary functions and activation rounding sites.""" +import numpy as np +from fp64_reference import bf + + +_ELEMENTARY_KERNELS = {} + + +def elementary(values, operation, backend='numpy'): + values = np.asarray(values, dtype=np.float32) + if backend == 'numpy': + if operation == 'rsqrt': + return (1 / np.sqrt(values.astype(np.float64))).astype(np.float32) + return getattr(np, operation)(values) + if backend != 'metal' or operation not in ('exp', 'exp2', 'cos', 'sin', 'rsqrt'): + raise ValueError('Unsupported elementary function backend or operation') + import mlx.core as mx + if operation not in _ELEMENTARY_KERNELS: + function = 'metal::exp2' if operation == 'exp2' else 'metal::precise::' + operation + _ELEMENTARY_KERNELS[operation] = mx.fast.metal_kernel( + name='lfm3_reference_elementary_' + operation, input_names=['X'], output_names=['Y'], + source='uint i=thread_position_in_grid.x;if(i> shifts.astype(np.uint64) + cross = shifts + bits > 32 + values[:, cross] |= padded[:, indices[cross] + 1] << (32 - shifts[cross]).astype(np.uint64) + codes = (values & ((1 << bits) - 1)).astype(np.float32) + scale = self.checkpoint.tensor(name + '.scales', rows).astype(np.float32).repeat(group, axis=-1) + bias = self.checkpoint.tensor(name + '.biases', rows).astype(np.float32).repeat(group, axis=-1) + return codes * scale + bias + + def linear(self, x, name): + count = self.checkpoint.files[self.checkpoint.map[name + '.weight']][2][name + '.weight']['shape'][0] + y = np.empty((*x.shape[:-1], count), dtype=np.float32) + for first in range(0, count, 512): + y[..., first:first + 512] = x @ self.weights(name, slice(first, first + 512)).T + return self.cast(y) + + def rms(self, x, gain, epsilon): + mean = np.mean(x * x, axis=-1, keepdims=True, dtype=np.float32) + inverse = np.float32(1) / np.sqrt(mean + np.float32(epsilon)) + return self.cast(self.cast(x * inverse) * gain) + + def norm(self, x, name): + return self.rms(x, self.tensor(name + '.weight'), self.text['norm_eps']) + + def rotary(self, x, kind): + spec = self.text['rope_parameters'][kind] + pairs = int(x.shape[-1] * spec.get('partial_rotary_factor', 1)) // 2 + power = -np.arange(pairs, dtype=np.float32) * np.float32(2 / x.shape[-1]) + frequency = np.concatenate((np.power(np.float32(spec['rope_theta']), power), + np.zeros(x.shape[-1] // 2 - pairs, dtype=np.float32))) + angle = np.arange(len(x), dtype=np.float32)[:, None] * frequency + angle = np.concatenate((angle, angle), axis=-1)[:, None] + first, second = np.split(x, 2, axis=-1) + return self.cast(x * self.cast(np.cos(angle)) + + np.concatenate((-second, first), axis=-1) * self.cast(np.sin(angle))) + + def attention_value(self, q, k, v, window=0, causal=True): + scores = (q @ k.swapaxes(-1, -2)) / np.float32(math.sqrt(q.shape[-1])) + if causal: + distance = np.arange(q.shape[1])[:, None] - np.arange(k.shape[1])[None] + allowed = distance >= 0 + if window: + allowed &= distance < window + scores = np.where(allowed, scores, np.float32(-np.inf)) + probability = np.exp(scores - scores.max(axis=-1, keepdims=True)) + probability = probability / probability.sum(axis=-1, keepdims=True, dtype=np.float32) + return self.cast(probability @ v) + + def attention(self, x, prefix, kind): + heads, kv, width = (self.text[k] for k in ('num_attention_heads', 'num_key_value_heads', 'head_dim')) + q = self.linear(x, prefix + 'self_attn.q_proj').reshape(len(x), heads, width) + k = self.linear(x, prefix + 'self_attn.k_proj').reshape(len(x), kv, width) + v = self.linear(x, prefix + 'self_attn.v_proj').reshape(len(x), kv, width).transpose(1, 0, 2) + q = self.rotary(self.norm(q, prefix + 'self_attn.q_norm'), kind).transpose(1, 0, 2) + k = self.rotary(self.norm(k, prefix + 'self_attn.k_norm'), kind).transpose(1, 0, 2) + k, v = np.repeat(k, heads // kv, axis=0), np.repeat(v, heads // kv, axis=0) + window = self.text['sliding_window'] if kind == 'sliding_attention' else 0 + value = self.attention_value(q, k, v, window).transpose(1, 0, 2) + gate = self.linear(x, prefix + 'self_attn.gate_proj') + gate = self.cast(np.float32(1) / (np.float32(1) + np.exp(-gate))) + return self.linear(self.cast(value * gate[:, :, None]).reshape(len(x), heads * width), + prefix + 'self_attn.o_proj') + + def conv(self, x, prefix): + left, gate, value = np.split(self.linear(x, prefix + 'conv.in_proj'), 3, axis=-1) + mixed = self.cast(left * value) + kernel = self.tensor(prefix + 'conv.conv.weight')[:, :, 0] + padded = np.pad(mixed, ((kernel.shape[1] - 1, 0), (0, 0))) + result = np.zeros_like(mixed) + for tap in range(kernel.shape[1]): + result = result + padded[tap:tap + len(x)] * kernel[:, tap] + return self.linear(self.cast(self.cast(result) * gate), prefix + 'conv.out_proj') + + def swiglu(self, gate, up): + return self.cast(self.cast(gate / (np.float32(1) + np.exp(-gate))) * up) + + def run(self, tokens, image_rows=None, image_start=None): + hidden = self.cast(self.weights(self.prefix + 'embed_tokens', tokens)) + if image_rows is not None: + rows = np.asarray(image_rows, dtype=np.float32) + if image_start is None or rows.ndim != 2 or rows.shape[1] != hidden.shape[1]: + raise ValueError('Expected projected image rows and a matching start') + expected = np.zeros(len(tokens), dtype=bool) + expected[image_start:image_start + len(rows)] = True + if image_start < 0 or image_start + len(rows) > len(tokens) or not np.array_equal(tokens == 125030, expected): + raise ValueError('Image span does not match placeholders') + hidden[image_start:image_start + len(rows)] = self.cast(rows) + elif image_start is not None: + raise ValueError('Image start requires projected rows') + self.take('embedding', hidden) + for index, kind in enumerate(self.text['layer_types']): + prefix = self.prefix + f'layers.{index}.' + x = self.norm(hidden, prefix + 'operator_norm') + result = self.conv(x, prefix) if kind == 'conv' else self.attention(x, prefix, kind) + hidden = self.cast(hidden + self.norm(result, prefix + 'post_operator_norm')) + x = self.norm(hidden, prefix + 'ffn_norm') + gate, up = self.linear(x, prefix + 'feed_forward.w1'), self.linear(x, prefix + 'feed_forward.w3') + result = self.linear(self.swiglu(gate, up), prefix + 'feed_forward.w2') + hidden = self.cast(hidden + self.norm(result, prefix + 'post_ffn_norm')) + self.take(f'text_layer_{index:02d}', hidden) + print('text_layer', index, flush=True) + return self.linear(self.norm(hidden, self.prefix + 'embedding_norm'), self.prefix + 'embed_tokens') + + +class Vision(Graph): + def __init__(self, directory, boundaries='fp32-bf16', trace=None): + super().__init__(directory, boundaries, trace) + self.vision = self.config['vision_config'] + + def dense(self, x, name): + y = x @ self.tensor(name + '.weight').T + if name + '.bias' in self.checkpoint.map: + y = y + self.tensor(name + '.bias') + return self.cast(y) + + def norm(self, x, name): + return self.rms(x, self.tensor(name + '.weight'), self.vision['norm_eps']) + + def conv(self, x, prefix, height, width): + left, gate, value = np.split(self.dense(x, prefix + 'conv.in_proj'), 3, axis=-1) + channels = self.vision['hidden_size'] + grid = self.cast(left * value).reshape(height // 2, width // 2, 2, 2, channels) + grid = grid.transpose(0, 2, 1, 3, 4).reshape(height, width, channels) + padded = np.pad(grid, ((1, 1), (1, 1), (0, 0))) + kernel = self.tensor(prefix + 'conv.spatial_conv.weight')[:, :, :, 0] + result = np.zeros_like(grid) + for row in range(3): + for col in range(3): + result = result + padded[row:row + height, col:col + width] * kernel[:, row, col] + merged = self.cast(result).reshape(height // 2, 2, width // 2, 2, channels) + merged = merged.transpose(0, 2, 1, 3, 4).reshape(height * width, channels) + return self.dense(self.cast(merged * gate), prefix + 'conv.out_proj') + + def rotary(self, x, coordinates): + spatial = x.shape[-1] // 2 + powers = -np.arange(0, spatial, 2, dtype=np.float32) / np.float32(spatial) + frequency = np.power(np.float32(self.vision['rope_parameters']['rope_theta']), powers) + vertical = coordinates[:, 0, None].astype(np.float32) * frequency + horizontal = coordinates[:, 1, None].astype(np.float32) * frequency + angle = np.concatenate((vertical, horizontal, vertical, horizontal), axis=-1)[:, None] + first, second = np.split(x, 2, axis=-1) + return self.cast(x * np.cos(angle) + np.concatenate((-second, first), axis=-1) * np.sin(angle)) + + def attention(self, x, prefix, coordinates): + heads, kv = (self.vision[k] for k in ('num_attention_heads', 'num_key_value_heads')) + dim = self.vision['hidden_size'] // heads + q = self.dense(x, prefix + 'self_attn.q_proj').reshape(len(x), heads, dim) + k = self.dense(x, prefix + 'self_attn.k_proj').reshape(len(x), kv, dim) + v = self.dense(x, prefix + 'self_attn.v_proj').reshape(len(x), kv, dim).transpose(1, 0, 2) + q = self.rotary(self.norm(q, prefix + 'self_attn.q_layernorm'), coordinates).transpose(1, 0, 2) + k = self.rotary(self.norm(k, prefix + 'self_attn.k_layernorm'), coordinates).transpose(1, 0, 2) + k, v = np.repeat(k, heads // kv, axis=0), np.repeat(v, heads // kv, axis=0) + value = self.attention_value(q, k, v, causal=False).transpose(1, 0, 2).reshape(len(x), heads * dim) + return self.dense(value, prefix + 'self_attn.out_proj') + + def run(self, patches, height, width): + if height <= 0 or width <= 0 or height % 2 or width % 2 or patches.shape != (height * width, 1536): + raise ValueError('Expected an even patch grid and1536 elements per patch') + coordinates = np.asarray([(h * 2 + a, w * 2 + b) for h in range(height // 2) + for w in range(width // 2) for a in range(2) for b in range(2)]) + hidden = self.norm(self.dense(self.cast(patches), 'vision_tower.patchifier'), 'vision_tower.embedding_norm') + for index, kind in enumerate(self.vision['layer_types']): + prefix = f'vision_tower.layers.{index}.' + x = self.norm(hidden, prefix + 'operator_norm') + result = self.conv(x, prefix, height, width) if kind == 'conv' else self.attention(x, prefix, coordinates) + hidden = self.cast(hidden + result) + x = self.norm(hidden, prefix + 'ffn_norm') + gate, up = self.dense(x, prefix + 'feed_forward.w1'), self.dense(x, prefix + 'feed_forward.w3') + hidden = self.cast(hidden + self.dense(self.swiglu(gate, up), prefix + 'feed_forward.w2')) + self.take(f'vision_layer_{index:02d}', hidden) + print('vision_layer', index, flush=True) + prefix = 'multi_modal_projector.' + centered = hidden - hidden.mean(axis=-1, keepdims=True, dtype=np.float32) + inverse = np.float32(1) / np.sqrt(np.mean(centered * centered, axis=-1, keepdims=True, dtype=np.float32) + + np.float32(self.config['projector_layer_norm_eps'])) + hidden = self.cast(centered * inverse * self.tensor(prefix + 'layer_norm.weight') + + self.tensor(prefix + 'layer_norm.bias')) + first = self.dense(hidden.reshape(-1, self.vision['hidden_size'] * self.config['downsample_factor'] ** 2), + prefix + 'linear_1') + argument = first / np.sqrt(np.float32(2)) + erf = np.fromiter((math.erf(float(v)) for v in argument.flat), dtype=np.float32).reshape(first.shape) + return self.dense(self.cast(np.float32(0.5) * first * (np.float32(1) + erf)), prefix + 'linear_2') + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--model-dir', type=Path, required=True) + p.add_argument('--tokens', type=Path, required=True) + p.add_argument('--output', type=Path, required=True) + p.add_argument('--boundaries', choices=('fp32', 'fp32-bf16'), required=True) + p.add_argument('--patches', type=Path) + p.add_argument('--grid-h', type=int) + p.add_argument('--grid-w', type=int) + p.add_argument('--image-rows', type=Path) + p.add_argument('--image-start', type=int) + args = p.parse_args() + tokens = np.load(args.tokens, allow_pickle=False) + model = Graph(args.model_dir, args.boundaries) + if tokens.ndim != 1 or not np.issubdtype(tokens.dtype, np.integer) or len(tokens) == 0: + p.error('Expected nonempty integer tokens') + if tokens.min() < 0 or tokens.max() >= model.text['vocab_size']: + p.error('Token IDs exceed vocabulary') + if args.patches and args.image_rows: + p.error('Choose own vision or fixed projected rows') + rows = np.load(args.image_rows, allow_pickle=False) if args.image_rows else None + if args.patches: + rows = Vision(args.model_dir, args.boundaries).run(np.load(args.patches, allow_pickle=False), + args.grid_h, args.grid_w) + np.save(args.output.with_suffix('.image.npy'), rows) + logits = model.run(tokens, rows, args.image_start) + if not np.isfinite(logits).all(): + raise ValueError('Nonfinite FP32 reference logits') + np.save(args.output, logits) + rounded = bf(logits).astype(np.float32) + args.output.with_suffix('.bf16').write_bytes((rounded.view(np.uint32) >> 16).astype('> np.uint64(48)) & np.uint64(0x8000)).astype(np.uint16) + exponent = ((bits >> np.uint64(52)) & np.uint64(0x7ff)).astype(np.int32) + fraction = bits & np.uint64((1 << 52) - 1) + significant = fraction | np.uint64(1 << 52) + high = significant >> np.uint64(45) + tail = significant & np.uint64((1 << 45) - 1) + increment = (tail > np.uint64(1 << 44)) | ((tail == np.uint64(1 << 44)) & ((high & 1) != 0)) + rounded = high + increment.astype(np.uint64) + encoded = ((exponent.astype(np.int64) - 897) * 128 + rounded.astype(np.int64)) + encoded = np.clip(encoded, 0, 0x7f80).astype(np.uint16) + small = np.abs(values) < np.float64(2.0 ** -126) + with np.errstate(over='ignore', invalid='ignore'): + subnormal = np.rint(np.where(small, np.abs(values), 0) * np.float64(2.0 ** 133)).astype(np.uint16) + encoded = np.where(small, subnormal, encoded).astype(np.uint16) + encoded = np.where(exponent == 0x7ff, np.where(fraction != 0, 0x7fc0, 0x7f80), encoded).astype(np.uint16) + return ((encoded | sign).astype(np.uint32) << 16).view(np.float32).astype(np.float64) + + +def inject_image_rows(hidden, tokens, rows, start): + rows = np.asarray(rows, dtype=np.float64) + if start is None or start < 0 or rows.ndim != 2 or not len(rows): + raise ValueError('Expected nonempty projected image rows and nonnegative image start') + if rows.shape[1] != hidden.shape[1] or start + len(rows) > len(hidden): + raise ValueError('Projected image rows exceed decoder geometry') + expected = np.zeros(len(tokens), dtype=bool) + expected[start:start + len(rows)] = True + if not np.array_equal(np.asarray(tokens) == 125030, expected): + raise ValueError('Image placeholder rows do not match projected image span') + if not np.isfinite(rows).all(): + raise ValueError('Projected image rows must be finite') + result = hidden.copy() + result[start:start + len(rows)] = bf(rows) + return result + + +class Oracle: + def __init__(self, directory, weight_mode='exact', dump=None, projection_rounding='fp64', + norm_conv_rounding='fp64', attention_rounding='fp64', elementary_backend='numpy'): + self.directory, self.weight_mode, self.dump = directory, weight_mode, dump + if attention_rounding not in ('fp64', 'metal-fp32') or elementary_backend not in ('numpy', 'metal'): + raise ValueError('Unsupported attention or elementary rounding mode') + self.attention_rounding, self.elementary_backend = attention_rounding, elementary_backend + self.projection_rounding = projection_rounding + if norm_conv_rounding not in ('fp64', 'metal'): + raise ValueError('Unsupported normalization/convolution rounding mode') + self.norm_conv_rounding = norm_conv_rounding + valid_rounding = projection_rounding in ('fp64', 'metal-row') + if not valid_rounding or (projection_rounding == 'metal-row' and weight_mode != 'exact'): + raise ValueError('Metal row replay requires exact affine weights') + directory = directory.resolve() + self.config = json.loads((directory / 'config.json').read_text()) + self.text = self.config.get('text_config', self.config) + if self.text.get('model_type') != 'lfm3' or not self.text.get('tie_word_embeddings'): + raise ValueError('Expected an LFM3 decoder with a tied vocabulary embedding') + index = json.loads((directory / 'model.safetensors.index.json').read_text()) + self.map, self.files = index['weight_map'], {} + for name in set(self.map.values()): + path = (directory / name).resolve() + if not path.is_relative_to(directory): + raise ValueError('Checkpoint shard escapes the model directory') + with path.open('rb') as handle: + length = struct.unpack('> shifts.astype(np.uint64) + cross = shifts + bits > 32 + values[:, cross] |= padded[:, indices[cross] + 1] << (32 - shifts[cross]).astype(np.uint64) + codes = (values & ((1 << bits) - 1)).astype(np.float64) + scale = self.tensor(name + '.scales', rows).repeat(group, axis=-1) + bias = self.tensor(name + '.biases', rows).repeat(group, axis=-1) + weight = scale * codes + bias + return bf(weight) if self.weight_mode == 'bf16' else weight + + def linear(self, x, name): + if self.projection_rounding == 'metal-row': + from row_projection_reference import affine_rows + return bf(affine_rows(self, name, x)) + path, base, header = self.files[self.map[name + '.weight']] + count = header[name + '.weight']['shape'][0] + output = np.empty((*x.shape[:-1], count), dtype=np.float64) + for start in range(0, count, 512): + end = min(start + 512, count) + output[..., start:end] = x @ self.weights(name, slice(start, end)).T + return bf(output) + + def norm(self, values, name): + if self.norm_conv_rounding == 'metal': + from text_rounding_reference import norm_rows + return norm_rows(values, self.tensor(name + '.weight'), self.text['norm_eps'], + self.elementary_backend if self.attention_rounding == 'metal-fp32' else None) + inverse = 1 / np.sqrt(np.mean(values * values, axis=-1, keepdims=True) + self.text['norm_eps']) + return bf(bf(values * inverse) * self.tensor(name + '.weight')) + + def take(self, name, value): + if self.dump: + self.dump(name, value) + + def rotary(self, values, kind): + if getattr(self, 'attention_rounding', 'fp64') == 'metal-fp32': + from text_attention_rounding import rotary + return rotary(values, self.text['rope_parameters'][kind], self.elementary_backend) + width = values.shape[-1] + spec = self.text['rope_parameters'][kind] + pairs = int(width * spec.get('partial_rotary_factor', 1)) // 2 + frequencies = np.concatenate((np.power(float(spec['rope_theta']), -np.arange(pairs) * 2 / width), + np.zeros(width // 2 - pairs))) + angle = np.arange(len(values))[:, None] * frequencies + angle = np.concatenate((angle, angle), axis=-1)[:, None, :] + cosine, sine = bf(np.cos(angle)), bf(np.sin(angle)) + a, b = np.split(values, 2, axis=-1) + return bf(values * cosine + np.concatenate((-b, a), axis=-1) * sine) + + def conv(self, x, prefix, tag): + projected = self.linear(x, prefix + 'conv.in_proj') + self.take(tag + '_conv_in', projected) + b, gate, value = np.split(projected, 3, axis=-1) + gated = bf(b * value) + kernel = self.tensor(prefix + 'conv.conv.weight')[:, :, 0] + taps = kernel.shape[1] + padded = np.pad(gated, ((taps - 1, 0), (0, 0))) + total = np.zeros_like(gated) + for tap in range(taps): + total = total + padded[tap:tap + len(x)] * kernel[:, tap] + if self.norm_conv_rounding == 'metal': + total = total.astype(np.float32).astype(np.float64) + mixed = bf(bf(total) * gate) + self.take(tag + '_conv_gated', mixed) + return self.linear(mixed, prefix + 'conv.out_proj') + + def attention(self, x, prefix, kind): + heads, kv, width = (self.text[key] for key in ('num_attention_heads', 'num_key_value_heads', 'head_dim')) + q = self.linear(x, prefix + 'self_attn.q_proj').reshape(len(x), heads, width) + k = self.linear(x, prefix + 'self_attn.k_proj').reshape(len(x), kv, width) + v = self.linear(x, prefix + 'self_attn.v_proj').reshape(len(x), kv, width) + q = self.rotary(self.norm(q, prefix + 'self_attn.q_norm'), kind).transpose(1, 0, 2) + k = self.rotary(self.norm(k, prefix + 'self_attn.k_norm'), kind).transpose(1, 0, 2) + v = v.transpose(1, 0, 2) + k, v = np.repeat(k, heads // kv, axis=0), np.repeat(v, heads // kv, axis=0) + if self.attention_rounding == 'metal-fp32': + from text_attention_rounding import attention + output = attention(q, k, v, self.text['sliding_window'] if kind == 'sliding_attention' else 0, + self.elementary_backend).transpose(1, 0, 2) + else: + scores = (q @ k.swapaxes(-1, -2)) / math.sqrt(width) + distance = np.arange(len(x))[:, None] - np.arange(len(x))[None, :] + allowed = distance >= 0 + if kind == 'sliding_attention': + allowed &= distance < self.text['sliding_window'] + scores = np.where(allowed, scores, -np.inf) + probabilities = np.exp(scores - scores.max(axis=-1, keepdims=True)) + probabilities /= probabilities.sum(axis=-1, keepdims=True) + output = bf((probabilities @ v).transpose(1, 0, 2)) + raw_gate = self.linear(x, prefix + 'self_attn.gate_proj') + if self.attention_rounding == 'metal-fp32': + from elementary_reference import elementary + gates = bf(np.float32(1) / (np.float32(1) + + elementary(-raw_gate.astype(np.float32), 'exp', self.elementary_backend))) + else: + gates = bf(1 / (1 + np.exp(-raw_gate))) + return self.linear(bf(output * gates[:, :, None]).reshape(len(x), heads * width), prefix + 'self_attn.o_proj') + + def run(self, tokens, image_rows=None, image_start=None): + hidden = bf(self.weights(self.prefix + 'embed_tokens', tokens)) + if image_rows is not None: + hidden = inject_image_rows(hidden, tokens, image_rows, image_start) + elif image_start is not None: + raise ValueError('Image start requires projected image rows') + self.take('embedding', hidden) + for index, kind in enumerate(self.text['layer_types']): + tag = f'layer_{index:02d}' + prefix = self.prefix + f'layers.{index}.' + x = self.norm(hidden, prefix + 'operator_norm') + self.take(tag + '_operator_norm', x) + result = self.conv(x, prefix, tag) if kind == 'conv' else self.attention(x, prefix, kind) + self.take(tag + '_mixer', result) + hidden = bf(hidden + self.norm(result, prefix + 'post_operator_norm')) + self.take(tag + '_post_operator', hidden) + x = self.norm(hidden, prefix + 'ffn_norm') + self.take(tag + '_ffn_norm', x) + gate, up = self.linear(x, prefix + 'feed_forward.w1'), self.linear(x, prefix + 'feed_forward.w3') + self.take(tag + '_gate', gate) + self.take(tag + '_up', up) + if self.attention_rounding == 'metal-fp32': + from elementary_reference import swiglu + act = swiglu(gate, up, self.elementary_backend) + else: + act = bf(bf(gate / (1 + np.exp(-gate))) * up) + self.take(tag + '_act', act) + result = self.linear(act, prefix + 'feed_forward.w2') + self.take(tag + '_down', result) + hidden = bf(hidden + self.norm(result, prefix + 'post_ffn_norm')) + self.take(tag + '_output', hidden) + print(tag, kind, flush=True) + final = self.norm(hidden, self.prefix + 'embedding_norm') + self.take('final_norm', final) + return self.linear(final, self.prefix + 'embed_tokens') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--model-dir', type=Path, required=True) + parser.add_argument('--tokens', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--weight-mode', choices=('exact', 'bf16'), default='exact') + parser.add_argument('--projection-rounding', choices=('fp64', 'metal-row'), default='fp64') + parser.add_argument('--attention-rounding', choices=('fp64', 'metal-fp32'), default='fp64') + parser.add_argument('--elementary-backend', choices=('numpy', 'metal'), default='numpy') + parser.add_argument('--norm-conv-rounding', choices=('fp64', 'metal'), default='fp64') + parser.add_argument('--trace-dir', type=Path) + parser.add_argument('--image-rows', type=Path, help='Projected rows as a two-dimensional numeric NumPy array') + parser.add_argument('--image-start', type=int) + args = parser.parse_args() + def write(path, values): + path.write_bytes((np.asarray(values, dtype=np.float32).view(np.uint32) >> 16).astype('= oracle.text['vocab_size']: + parser.error('Token IDs exceed the decoder vocabulary') + if tokens.size > oracle.text['max_position_embeddings']: + parser.error('Token array exceeds the decoder context') + image_rows = np.load(args.image_rows, allow_pickle=False) if args.image_rows else None + if (args.image_rows is None) != (args.image_start is None): + parser.error('--image-rows and --image-start must be supplied together') + logits = oracle.run(tokens, image_rows, args.image_start) + if not np.isfinite(logits).all(): + raise ValueError('Reference produced nonfinite logits') + write(args.output, logits) + arithmetic = 'FP64 reductions with direct binary64-to-BF16 RNE activation boundaries' + if args.projection_rounding == 'metal-row': + arithmetic += '; affine projections instead replay the Metal FP32 lane rounding, other operations remain FP64' + if args.norm_conv_rounding == 'metal': + arithmetic += '; normalization and convolution replay the production FP32 reduction sites' + if args.attention_rounding == 'metal-fp32': + arithmetic += ('; bounded decoder attention/rotary/activations replay FP32 sites with explicit ' + + args.elementary_backend + ' elementary functions') + metadata = {'dtype': 'BF16', 'arithmetic': arithmetic, 'weight_mode': args.weight_mode, + 'attention_rounding': args.attention_rounding, 'elementary_backend': args.elementary_backend, + 'projection_rounding': args.projection_rounding, 'norm_conv_rounding': args.norm_conv_rounding, + 'shape': list(logits.shape), + 'accepted_threshold': None} + args.output.with_suffix('.json').write_text(json.dumps(metadata, indent=2) + '\n') + +if __name__ == '__main__': + main() diff --git a/tools/lfm3/inspect_checkpoint.py b/tools/lfm3/inspect_checkpoint.py new file mode 100644 index 0000000000..2fec63457d --- /dev/null +++ b/tools/lfm3/inspect_checkpoint.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +"""Inspect pinned Hugging Face configs and safetensors headers without downloading tensor data.""" + +import argparse +import collections +import json +import math +import os +import re +import struct +import sys +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path, PurePosixPath + +MAX_JSON = 16 * 1024 * 1024 +MAX_HEADER = 16 * 1024 * 1024 + + +class InspectionError(Exception): + pass + + +def safe_name(name): + path = PurePosixPath(name) + if not name or path.is_absolute() or '..' in path.parts or '\\' in name: + raise InspectionError(f'Unsafe checkpoint file name: {name!r}') + return name + + +def read_json_bytes(data, description): + try: + return json.loads(data) + except (ValueError, UnicodeError) as exc: + raise InspectionError(f'Invalid JSON in {description}: {exc}') from exc + + +class Remote: + def __init__(self, repository, revision): + if not re.fullmatch(r'[\w.-]+/[\w.-]+', repository): + raise InspectionError('Repository must have owner/name form') + api = 'https://huggingface.co/api/models/' + repository + '/revision/' + metadata = self.json_url(api + urllib.parse.quote(revision, safe='')) + self.revision = metadata.get('sha', '') + if not re.fullmatch(r'[0-9a-f]{40}', self.revision): + raise InspectionError('Hugging Face did not resolve a full commit revision') + self.repository = repository + self.files = {entry['rfilename'] for entry in metadata.get('siblings', [])} + self.base = f'https://huggingface.co/{repository}/resolve/{self.revision}/' + + @staticmethod + def fetch(url, limit, byte_range=None, with_total=False): + headers = {'User-Agent': 'TensorFold-checkpoint-inspector', 'Accept-Encoding': 'identity'} + if byte_range is not None: + start, end = byte_range + headers['Range'] = f'bytes={start}-{end}' + try: + with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=60) as response: + if byte_range is not None: + match = re.fullmatch(r'bytes (\d+)-(\d+)/(\d+)', response.headers.get('Content-Range', '')) + if response.status != 206 or not match: + raise InspectionError('Server refused an exact HTTP byte range; refusing tensor download') + first, last, total = map(int, match.groups()) + if (first, last) != byte_range or total <= last: + raise InspectionError('Server returned an unexpected byte range') + if response.headers.get('Content-Encoding', 'identity') != 'identity': + raise InspectionError('Compressed range response is not supported') + length = response.headers.get('Content-Length') + if length is not None and int(length) > limit: + raise InspectionError(f'Response exceeds {limit}-byte body limit') + data = response.read(limit + 1) + if len(data) > limit: + raise InspectionError(f'Response exceeds {limit}-byte body limit') + if byte_range is not None and len(data) != byte_range[1] - byte_range[0] + 1: + raise InspectionError('Truncated byte-range response') + return (data, total) if with_total else data + except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc: + raise InspectionError(f'Cannot fetch {url}: {exc}') from exc + + @classmethod + def json_url(cls, url): + return read_json_bytes(cls.fetch(url, MAX_JSON), url) + + def json(self, name): + return self.json_url(self.base + urllib.parse.quote(safe_name(name), safe='/')) + + def header(self, name): + url = self.base + urllib.parse.quote(safe_name(name), safe='/') + prefix, total = self.fetch(url, 8, (0, 7), with_total=True) + size = header_size(prefix) + if total < 8 + size: + raise InspectionError('Safetensors header extends beyond shard size') + data, second_total = self.fetch(url, size, (8, 7 + size), with_total=True) + if total != second_total: + raise InspectionError('Shard size changed between header requests') + return read_json_bytes(data, name), total - 8 - size + + +class Local: + def __init__(self, directory): + self.directory = Path(directory).resolve() + self.repository = str(self.directory) + self.revision = None + self.files = {str(path.relative_to(self.directory)) for path in self.directory.rglob('*.safetensors')} + if (self.directory / 'model.safetensors.index.json').is_file(): + self.files.add('model.safetensors.index.json') + + def path(self, name): + path = (self.directory / safe_name(name)).resolve() + if not path.is_relative_to(self.directory): + raise InspectionError('Checkpoint file escapes local directory') + return path + + def json(self, name): + with self.path(name).open('rb') as stream: + data = stream.read(MAX_JSON + 1) + if len(data) > MAX_JSON: + raise InspectionError('Local JSON exceeds body limit') + return read_json_bytes(data, name) + + def header(self, name): + with self.path(name).open('rb') as stream: + size = header_size(stream.read(8)) + data = stream.read(size) + total = os.fstat(stream.fileno()).st_size + if len(data) != size: + raise InspectionError('Truncated local safetensors header') + return read_json_bytes(data, name), total - 8 - size + + +def header_size(prefix): + if len(prefix) != 8: + raise InspectionError('Truncated safetensors size prefix') + size = struct.unpack(' mc)), + 'columns_native_better': int(np.count_nonzero(nc < mc)), + 'worst_column_increases': [{'column': int(i), 'native_rms': float(nc[i]), + 'mlx_rms': float(mc[i]), 'increase': float(increase[i])} for i in worst], + }) + np.savez_compressed(output / (case['name'] + '-' + label + '-columns.npz'), native=nc, mlx=mc) + return { + 'name': case['name'], 'model': case['model_directory'], 'module': case['module'], + 'shape': [len(x), case['k'], case['n']], 'quantization': list(model.quant(case['module'])), + 'input_sha256': hashlib.sha256((inputs / case['input_file']).read_bytes()).hexdigest(), + 'native_prompt_sha256': hashlib.sha256((candidate / (case['name'] + '.bf16')).read_bytes()).hexdigest(), + 'native_decode_sha256': hashlib.sha256((candidate / (case['name'] + '-one.bf16')).read_bytes()).hexdigest(), + 'native_prompt_decode_equal': bool(np.array_equal(native[:1], one)), + 'mlx_prompt_decode_equal': bool(np.array_equal(mlx[:1], mlx_one)), 'comparisons': records, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--models', type=Path, required=True) + parser.add_argument('--inputs', type=Path, required=True) + parser.add_argument('--candidate', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + import mlx.core as mx + args.output.mkdir(parents=True, exist_ok=True) + cases = json.loads((args.inputs / 'manifest.json').read_text()) + records, current = [], None + for case in cases: + if current != case['model_directory']: + if current is not None: + del reference, model + mx.clear_cache() + current = case['model_directory'] + model = Oracle(args.models / current) + reference = Reference(args.models / current, mx) + record = compare(case, model, reference, args.inputs, args.candidate, args.output, mx) + records.append(record) + (args.output / 'results.json').write_text(json.dumps(records, indent=2) + '\n') + print(json.dumps(record), flush=True) + summary = { + 'cells': len(records), 'projection_outputs': sum(c['shape'][0] * c['shape'][2] for c in records), + 'native_prompt_decode_equal_cells': sum(c['native_prompt_decode_equal'] for c in records), + 'mlx_prompt_decode_equal_cells': sum(c['mlx_prompt_decode_equal'] for c in records), + 'acceptance_threshold': None, + } + (args.output / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n') + print(json.dumps(summary), flush=True) + + +if __name__ == '__main__': + main() diff --git a/tools/lfm3/reference.py b/tools/lfm3/reference.py new file mode 100644 index 0000000000..f536b6a98a --- /dev/null +++ b/tools/lfm3/reference.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +"""Developer-only MLX teacher-forced LFM3 text reference, independent of native kernels.""" + +import argparse +import json +import math +from pathlib import Path + + +def canonical(name): + if name.endswith('.weight'): + name = name[:-7] + for prefix in ('model.language_model.', 'language_model.', 'text_model.', 'model.'): + if name.startswith(prefix): + name = name[len(prefix):] + return name + + +def quantization(config, name): + spec = config.get('quantization', config.get('quantization_config')) + if not isinstance(spec, dict): + raise ValueError('Checkpoint needs affine quantization metadata') + width = spec + for key, value in spec.items(): + if isinstance(value, dict) and canonical(key) == canonical(name): + width = value + break + bits, group = width['bits'], width['group_size'] + if bits not in (4, 6, 8) or group not in (32, 64) or width.get('mode', 'affine') != 'affine': + raise ValueError(f'Unsupported quantization for {name}') + return bits, group + + +class Reference: + def __init__(self, directory, mx, attention='sdpa'): + self.mx = mx + self.attention_mode = attention + self.config = json.loads((directory / 'config.json').read_text()) + self.text = self.config.get('text_config', self.config) + if self.text.get('model_type') != 'lfm3' or not self.text.get('tie_word_embeddings'): + raise ValueError('Expected LFM3 with a tied output embedding') + self.weights = {} + index = directory / 'model.safetensors.index.json' + files = sorted(set(json.loads(index.read_text())['weight_map'].values())) if index.exists() else [ + path.name for path in directory.glob('*.safetensors') + ] + if not files: + raise ValueError('No safetensors checkpoint found') + for name in files: + path = (directory / name).resolve() + if not path.is_relative_to(directory.resolve()): + raise ValueError('Shard path escapes checkpoint directory') + for key, value in mx.load(str(path)).items(): + if key in self.weights: + raise ValueError(f'Duplicate tensor {key}') + self.weights[key] = value + wrapped = 'language_model.model.embed_tokens.weight' in self.weights + bare = 'model.embed_tokens.weight' in self.weights + if wrapped == bare: + raise ValueError('Checkpoint must use exactly one text namespace') + self.prefix = 'language_model.model.' if wrapped else 'model.' + + def round(self, x): + return x.astype(self.mx.bfloat16) + + def linear(self, x, module): + bits, group = quantization(self.config, module) + return self.round(self.mx.quantized_matmul( + x, self.weights[module + '.weight'], self.weights[module + '.scales'], + self.weights[module + '.biases'], transpose=True, group_size=group, bits=bits, + )) + + def embedding(self, tokens): + mx = self.mx + name = self.prefix + 'embed_tokens' + bits, group = quantization(self.config, name) + ids = mx.array(tokens, dtype=mx.int32) + return self.round(mx.dequantize( + mx.take(self.weights[name + '.weight'], ids, axis=0), + mx.take(self.weights[name + '.scales'], ids, axis=0), + mx.take(self.weights[name + '.biases'], ids, axis=0), group_size=group, bits=bits, + )) + + def norm(self, x, name): + mx = self.mx + values = x.astype(mx.float32) + scale = mx.rsqrt(mx.mean(values * values, axis=-1, keepdims=True) + self.text['norm_eps']) + normalized = self.round(values * scale) + return self.round(normalized * self.weights[name + '.weight']) + + def convolution(self, x, prefix, dump=None): + mx = self.mx + width = self.text['hidden_size'] + projected = self.linear(x, prefix + 'conv.in_proj') + if dump: + dump('conv_in', projected) + b, gate, values = mx.split(projected, 3, axis=-1) + gated = self.round(b * values) + kernel = self.weights[prefix + 'conv.conv.weight'].astype(mx.float32)[:, :, 0] + taps = kernel.shape[1] + padded = mx.pad(gated.astype(mx.float32), ((taps - 1, 0), (0, 0))) + total = mx.zeros((x.shape[0], width), dtype=mx.float32) + for tap in range(taps): + total = total + padded[tap:tap + x.shape[0]] * kernel[:, tap] + result = self.round(self.round(total) * gate) + if dump: + dump('conv_gated', result) + return self.linear(result, prefix + 'conv.out_proj') + + def rotary(self, x, kind): + mx = self.mx + dim = self.text['head_dim'] + spec = self.text['rope_parameters'][kind] + pairs = int(dim * spec.get('partial_rotary_factor', 1.0)) // 2 + frequencies = mx.power(mx.array(spec['rope_theta'], dtype=mx.float32), + -mx.arange(0, pairs * 2, 2, dtype=mx.float32) / dim) + frequencies = mx.concatenate((frequencies, mx.zeros((dim // 2 - pairs,), dtype=mx.float32))) + angles = mx.arange(x.shape[0], dtype=mx.float32)[:, None] * frequencies[None, :] + angles = mx.concatenate((angles, angles), axis=-1) + cosine = self.round(mx.cos(angles)).astype(mx.float32)[:, None, :] + sine = self.round(mx.sin(angles)).astype(mx.float32)[:, None, :] + values = x.astype(mx.float32) + first, second = mx.split(values, 2, axis=-1) + rotated = mx.concatenate((-second, first), axis=-1) + return self.round(values * cosine + rotated * sine) + + def attention(self, x, prefix, kind): + mx = self.mx + length = x.shape[0] + heads, kv, dim = (self.text[key] for key in ('num_attention_heads', 'num_key_value_heads', 'head_dim')) + q = self.linear(x, prefix + 'self_attn.q_proj').reshape(length, heads, dim) + k = self.linear(x, prefix + 'self_attn.k_proj').reshape(length, kv, dim) + v = self.linear(x, prefix + 'self_attn.v_proj').reshape(length, kv, dim) + q = self.rotary(self.norm(q, prefix + 'self_attn.q_norm'), kind).transpose(1, 0, 2)[None] + k = self.rotary(self.norm(k, prefix + 'self_attn.k_norm'), kind).transpose(1, 0, 2)[None] + v = v.transpose(1, 0, 2)[None] + positions = mx.arange(length) + distance = positions[:, None] - positions[None, :] + allowed = distance >= 0 + if kind == 'sliding_attention': + allowed = allowed & (distance < self.text['sliding_window']) + if self.attention_mode == 'sdpa': + output = mx.fast.scaled_dot_product_attention(q, k, v, scale=1 / math.sqrt(dim), mask=allowed) + else: + k = mx.repeat(k, heads // kv, axis=1) + v = mx.repeat(v, heads // kv, axis=1) + scores = self.round(mx.matmul(q, k.swapaxes(-1, -2))) + scores = self.round(scores * (1 / math.sqrt(dim))) + scores = mx.where(allowed, scores.astype(mx.float32), -float('inf')) + probabilities = self.round(mx.softmax(scores, axis=-1)) + output = mx.matmul(probabilities, v) + output = self.round(output[0].transpose(1, 0, 2)) + logits = self.linear(x, prefix + 'self_attn.gate_proj') + gate = self.round(mx.sigmoid(logits.astype(mx.float32))) + output = self.round(output * gate[:, :, None]).reshape(length, heads * dim) + return self.linear(output, prefix + 'self_attn.o_proj') + + def run(self, tokens, dump=None, last_only=False, image_rows=None, image_start=0): + mx = self.mx + hidden = self.embedding(tokens) + if image_rows is not None: + hidden = mx.concatenate((hidden[:image_start], image_rows, hidden[image_start + image_rows.shape[0]:]), axis=0) + if dump: + dump('embedding', hidden) + for index, kind in enumerate(self.text['layer_types']): + prefix = self.prefix + f'layers.{index}.' + x = self.norm(hidden, prefix + 'operator_norm') + if dump: + dump(f'layer_{index:02d}_operator_norm', x) + conv_dump = (lambda suffix, value: dump(f'layer_{index:02d}_{suffix}', value)) if dump else None + result = self.convolution(x, prefix, conv_dump) if kind == 'conv' else self.attention(x, prefix, kind) + if dump: + dump(f'layer_{index:02d}_mixer', result) + hidden = self.round(hidden + self.norm(result, prefix + 'post_operator_norm')) + if dump: + dump(f'layer_{index:02d}_post_operator', hidden) + x = self.norm(hidden, prefix + 'ffn_norm') + if dump: + dump(f'layer_{index:02d}_ffn_norm', x) + gate = self.linear(x, prefix + 'feed_forward.w1') + up = self.linear(x, prefix + 'feed_forward.w3') + if dump: + dump(f'layer_{index:02d}_gate', gate) + dump(f'layer_{index:02d}_up', up) + silu = self.round(gate.astype(mx.float32) * mx.sigmoid(gate.astype(mx.float32))) + act = self.round(silu * up) + if dump: + dump(f'layer_{index:02d}_act', act) + result = self.linear(act, prefix + 'feed_forward.w2') + if dump: + dump(f'layer_{index:02d}_down', result) + hidden = self.round(hidden + self.norm(result, prefix + 'post_ffn_norm')) + mx.eval(hidden) + if dump: + dump(f'layer_{index:02d}', hidden) + dump(f'layer_{index:02d}_output', hidden) + final = self.norm(hidden[-1:] if last_only else hidden, self.prefix + 'embedding_norm') + if dump: + dump('final_norm', final) + return self.linear(final, self.prefix + 'embed_tokens') + + +def write_bf16(path, values, mx, np): + floats = np.asarray(values.astype(mx.float32)) + raw = (floats.view(np.uint32) >> 16).astype('= model.text['vocab_size']: + parser.error('Token IDs exceed the checkpoint vocabulary') + if tokens.size > model.text['max_position_embeddings']: + parser.error('Token sequence exceeds the checkpoint context') + image_rows = None + image_count = 0 + if args.image_rows is not None: + if model.config.get('model_type') != 'lfm3_vl': + parser.error('Projected image rows require an LFM3-VL checkpoint') + width = model.text['hidden_size'] + if args.image_rows.stat().st_size == 0 or args.image_rows.stat().st_size % (width * 2): + parser.error('Expected complete BF16 projected image rows') + raw = np.fromfile(args.image_rows, dtype=' tokens.size or not np.isfinite(values).all(): + parser.error('Image rows are nonfinite or exceed the prompt') + positions = np.flatnonzero(tokens == model.config['image_token_id']) + if not np.array_equal(positions, np.arange(args.image_start, end)): + parser.error('Image placeholders must match the supplied row span exactly') + image_rows = mx.array(values).astype(mx.bfloat16) + elif args.image_start != 0: + parser.error('--image-start requires --image-rows') + intermediates = {} + + def dump(name, values): + write_bf16(args.intermediates_dir / (name + '.bf16'), values, mx, np) + intermediates[name] = {'shape': list(values.shape), 'dtype': 'BF16'} + + if args.intermediates_dir: + args.intermediates_dir.mkdir(parents=True, exist_ok=True) + logits = model.run(tokens, dump if args.intermediates_dir else None, args.last_only, image_rows, args.image_start) + mx.eval(logits) + write_bf16(args.output, logits, mx, np) + metadata = { + 'shape': list(logits.shape), 'dtype': 'BF16', 'byte_order': 'little', + 'attention': args.attention, 'token_count': int(tokens.size), 'intermediates': intermediates, + 'image_start': args.image_start, 'image_rows': image_count, + } + args.output.with_suffix(args.output.suffix + '.json').write_text(json.dumps(metadata, indent=2) + '\n') + print(json.dumps(metadata)) + + +if __name__ == '__main__': + main() diff --git a/tools/lfm3/row_projection_reference.py b/tools/lfm3/row_projection_reference.py new file mode 100644 index 0000000000..b8737a56c4 --- /dev/null +++ b/tools/lfm3/row_projection_reference.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Independent affine row-projection replay with the shared Metal helper's FP32 rounding sites.""" +import argparse +import ctypes +import os +import json +from pathlib import Path +import numpy as np +from fp64_reference import Oracle, bf + + +def affine_rows(oracle, name, inputs): + original = np.asarray(inputs) + if not np.isfinite(original).all() or not np.array_equal(original, bf(original)): + raise ValueError('Expected finite BF16-representable inputs') + inputs = np.asarray(original, dtype=np.float32) + bits, group = oracle.quant(name) + words = oracle.tensor(name + '.weight') + n = words.shape[0] + k = words.shape[1] * 32 // bits + if inputs.ndim != 2 or inputs.shape[1] != k or k % 512: + raise ValueError('Expected row-by-K inputs with a complete 512-value lane pass') + positions = np.arange(k) * bits + indices, shifts = positions // 32, positions % 32 + padded = np.pad(words.astype(np.uint64), ((0, 0), (0, 1))) + codes = padded[:, indices] >> shifts.astype(np.uint64) + cross = shifts + bits > 32 + codes[:, cross] |= padded[:, indices[cross] + 1] << (32 - shifts[cross]).astype(np.uint64) + codes = (codes & ((1 << bits) - 1)).astype(np.uint8) + scales = oracle.tensor(name + '.scales').astype(np.float32) + biases = oracle.tensor(name + '.biases').astype(np.float32) + output = np.empty((len(inputs), n), dtype=np.float32) + library = os.environ.get('LFM3_ROW_REPLAY_LIBRARY') + if library: + function = ctypes.CDLL(library).lfm3_row_replay + float_pointer, byte_pointer = ctypes.POINTER(ctypes.c_float), ctypes.POINTER(ctypes.c_uint8) + function.argtypes = [float_pointer, byte_pointer, float_pointer, float_pointer, float_pointer] + [ctypes.c_int] * 5 + function.restype = ctypes.c_int + inputs = np.ascontiguousarray(inputs) + codes = np.ascontiguousarray(codes) + scales, biases = np.ascontiguousarray(scales), np.ascontiguousarray(biases) + status = function(inputs.ctypes.data_as(float_pointer), codes.ctypes.data_as(byte_pointer), + scales.ctypes.data_as(float_pointer), biases.ctypes.data_as(float_pointer), + output.ctypes.data_as(float_pointer), len(inputs), n, k, group, bits) + if status != 0: + raise MemoryError('CPU projection replay scratch allocation failed') + return output + for first in range(0, len(inputs), 8): + x = inputs[first:first + 8] + for column in range(0, n, 256): + count = min(256, n - column) + lanes = np.zeros((len(x), count, 32), dtype=np.float32) + for at in range(0, k, 512): + xx = x[:, at:at + 512].reshape(len(x), 1, 32, 16) + qq = codes[column:column + count, at:at + 512].reshape(1, count, 32, 16) + sums = np.zeros((len(x), 1, 32), dtype=np.float32) + for i in range(16): + sums = sums + xx[..., i] + if bits == 4: + place = np.tile(np.array([1, 16, 256, 4096], dtype=np.float32), 4) + xx, qq = xx / place, qq * place + dot = np.zeros_like(lanes) + for i in range(0, 16, 4): + term = xx[..., i] * qq[..., i] + for j in range(1, 4): + term = term + xx[..., i + j] * qq[..., i + j] + dot = dot + term + groups = (np.arange(32) * 16 + at) // group + scale = scales[column:column + count, groups][None] + bias = biases[column:column + count, groups][None] + bias_product = sums * bias + fused = (scale.astype(np.float64) * dot.astype(np.float64) + bias_product).astype(np.float32) + lanes = lanes + fused + for distance in (1, 2, 4, 8, 16): + lanes = lanes + lanes[..., np.arange(32) ^ distance] + output[first:first + len(x), column:column + count] = lanes[..., 0] + return output + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--model-dir', type=Path, required=True) + parser.add_argument('--projection', required=True) + parser.add_argument('--inputs', type=Path, required=True, help='BF16-representable numeric NumPy rows') + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--native-raw', type=Path, help='Optional captured pre-BF16 FP32 NumPy output') + args = parser.parse_args() + result = affine_rows(Oracle(args.model_dir), args.projection, np.load(args.inputs, allow_pickle=False)) + np.save(args.output, result) + receipt = {'shape': list(result.shape), 'projection': args.projection, + 'scope': 'Projection rounding replay; not a whole-model fidelity pass', + 'affine_fusion': ('Explicit CPU fmaf' if os.environ.get('LFM3_ROW_REPLAY_LIBRARY') else + 'FP64 multiply-add then FP32 round; not arbitrary IEEE fmaf proof'), + 'accepted_threshold': None} + if args.native_raw: + native = np.load(args.native_raw, allow_pickle=False) + if native.shape != result.shape or native.dtype != np.float32: + raise ValueError('Captured native output must have the same FP32 geometry') + receipt.update(raw_equal=int(np.count_nonzero(native == result)), values=result.size, + bf16_equal=int(np.count_nonzero(bf(native) == bf(result))), + maximum_absolute_error=float(np.max(np.abs(native.astype(np.float64) - result)))) + args.output.with_suffix('.json').write_text(json.dumps(receipt, indent=2) + '\n') + + +def dense_rows(inputs, weight): + inputs, weight = np.asarray(inputs), np.asarray(weight) + if inputs.ndim != 2 or weight.ndim != 2 or inputs.shape[1] != weight.shape[1] or inputs.shape[1] % 32: + raise ValueError('Expected compatible dense row matrices with complete 32-lane passes') + if any(not np.isfinite(x).all() or not np.array_equal(x, bf(x)) for x in (inputs, weight)): + raise ValueError('Dense replay requires finite BF16 operands') + inputs, weight = np.ascontiguousarray(inputs, dtype=np.float32), np.ascontiguousarray(weight, dtype=np.float32) + output = np.empty((len(inputs), len(weight)), dtype=np.float32) + library = os.environ.get('LFM3_ROW_REPLAY_LIBRARY') + if library: + function = ctypes.CDLL(library).lfm3_dense_replay + pointer = ctypes.POINTER(ctypes.c_float) + function.argtypes = [pointer, pointer, pointer] + [ctypes.c_int] * 3 + function.restype = None + function(inputs.ctypes.data_as(pointer), weight.ctypes.data_as(pointer), output.ctypes.data_as(pointer), + len(inputs), len(weight), inputs.shape[1]) + return output + for row in range(0, len(inputs), 8): + x = inputs[row:row + 8] + for column in range(0, len(weight), 256): + w = weight[column:column + 256] + lanes = np.zeros((len(x), len(w), 32), dtype=np.float32) + for at in range(0, inputs.shape[1], 32): + lanes = (x[:, None, at:at + 32].astype(np.float64) + * w[None, :, at:at + 32].astype(np.float64) + lanes).astype(np.float32) + for distance in (1, 2, 4, 8, 16): + lanes = lanes + lanes[..., np.arange(32) ^ distance] + output[row:row + len(x), column:column + len(w)] = lanes[..., 0] + return output + + +if __name__ == '__main__': + main() diff --git a/tools/lfm3/row_projection_replay.c b/tools/lfm3/row_projection_replay.c new file mode 100644 index 0000000000..9eb61c54d0 --- /dev/null +++ b/tools/lfm3/row_projection_replay.c @@ -0,0 +1,77 @@ +// Optional CPU diagnostic accelerator for the independent Metal row-reduction replay, never native inference. +#include +#include +#include + +// Compile with -ffp-contract=off: only the explicitly named affine fmaf is contracted. +int lfm3_row_replay(const float *x, const uint8_t *codes, const float *scales, const float *biases, + float *output, int rows, int n, int k, int group, int bits) { + float *placed = malloc((int64_t)rows * k * sizeof(float)); + float *sums = malloc((int64_t)rows * (k / 16) * sizeof(float)); + if (!placed || !sums) { free(placed); free(sums); return 1; } + for (int row = 0; row < rows; ++row) { + for (int start = 0; start < k; start += 16) { + float sum = 0.0f; + for (int i = 0; i < 16; ++i) { + const int64_t at = (int64_t)row * k + start + i; + sum = sum + x[at]; + const float place = bits == 4 ? (float)(1u << (4 * (i % 4))) : 1.0f; + placed[at] = x[at] / place; + } + sums[(int64_t)row * (k / 16) + start / 16] = sum; + } + } + for (int row = 0; row < rows; ++row) { + for (int column = 0; column < n; ++column) { + float lanes[32] = {0}; + for (int base = 0; base < k; base += 512) { + for (int lane = 0; lane < 32; ++lane) { + const int start = base + lane * 16; + const float *input = placed + (int64_t)row * k + start; + const uint8_t *weight = codes + (int64_t)column * k + start; + const float sum = sums[(int64_t)row * (k / 16) + start / 16]; + float dot = 0.0f; + for (int i = 0; i < 16; i += 4) { + float products[4]; + for (int j = 0; j < 4; ++j) { + const float place = bits == 4 ? (float)(1u << (4 * j)) : 1.0f; + const float placed_input = input[i + j]; + const float placed_code = (float)weight[i + j] * place; + products[j] = placed_input * placed_code; + } + float four = products[0] + products[1]; + four = four + products[2]; + four = four + products[3]; + dot = dot + four; + } + const int64_t at = (int64_t)column * (k / group) + start / group; + const float bias_product = sum * biases[at]; + lanes[lane] = lanes[lane] + fmaf(scales[at], dot, bias_product); + } + } + for (int distance = 1; distance < 32; distance *= 2) + for (int lane = 0; lane < 32; lane += 2 * distance) + lanes[lane] = lanes[lane] + lanes[lane + distance]; + output[(int64_t)row * n + column] = lanes[0]; + } + } + free(placed); + free(sums); + return 0; +} + +// BF16 operands' dense projection uses one FP32 accumulator per lane and the same adjacent butterfly. +void lfm3_dense_replay(const float *x, const float *weight, float *output, int rows, int n, int k) { + for (int row = 0; row < rows; ++row) { + for (int column = 0; column < n; ++column) { + float lanes[32] = {0}; + for (int lane = 0; lane < 32; ++lane) + for (int i = lane; i < k; i += 32) + lanes[lane] = fmaf(x[(int64_t)row * k + i], weight[(int64_t)column * k + i], lanes[lane]); + for (int distance = 1; distance < 32; distance *= 2) + for (int lane = 0; lane < 32; lane += 2 * distance) + lanes[lane] = lanes[lane] + lanes[lane + distance]; + output[(int64_t)row * n + column] = lanes[0]; + } + } +} diff --git a/tools/lfm3/serve_check.py b/tools/lfm3/serve_check.py new file mode 100644 index 0000000000..feaf54fa02 --- /dev/null +++ b/tools/lfm3/serve_check.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +"""Run owned loopback servers to check LFM3 served token equality, prompt reuse and image behavior.""" + +import argparse +import base64 +import concurrent.futures +import hashlib +import json +import re +import socket +import struct +import subprocess +import sys +import tempfile +import threading +import time +import urllib.error +import urllib.request +import zlib +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from bench_evidence import Evidence, token_equal + + +def solid_png(red, green, blue): + def chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data)) + rows = (b'\0' + bytes((red, green, blue)) * 256) * 256 + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 256, 256, 8, 2, 0, 0, 0)) + chunk( + b'IDAT', zlib.compress(rows), + ) + chunk(b'IEND', b'') + + +def image_url(data): + return {'type': 'image_url', 'image_url': {'url': 'data:image/png;base64,' + base64.b64encode(data).decode()}} + + +class Server: + def __init__(self, binary, directory, cache, log_directory): + self.binary, self.directory, self.cache = binary, directory, cache + self.log_directory = log_directory + self.process = None + + def __enter__(self): + with socket.socket() as sock: + sock.bind(('127.0.0.1', 0)) + port = sock.getsockname()[1] + self.base = f'http://127.0.0.1:{port}' + self.log = tempfile.NamedTemporaryFile(mode='w+b', prefix='server-', suffix='.log', + dir=self.log_directory, delete=False) + command = [str(self.binary), 'serve', str(self.directory), '--host', '127.0.0.1', '--port', str(port), + '--name', 'lfm3-check', '--context', '2048', '--parallel', '4', '--temperature', '0', + '--prompt-cache-gib', str(self.cache), '--no-thinking', '--no-update-check'] + self.command = command + self.process = subprocess.Popen(command, stdout=self.log, stderr=subprocess.STDOUT) + try: + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + if self.process.poll() is not None: + raise RuntimeError(f'Server exited {self.process.returncode}; see {self.log.name}') + try: + with urllib.request.urlopen(self.base + '/v1/models', timeout=1) as response: + if response.status == 200: + return self + except (urllib.error.URLError, TimeoutError): + pass + time.sleep(0.2) + raise TimeoutError(f'Server startup timed out; see {self.log.name}') + except BaseException: + self.__exit__(None, None, None) + raise + + def __exit__(self, *_): + if self.process is not None and self.process.poll() is None: + self.process.terminate() + try: + self.process.wait(timeout=20) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=10) + self.log.close() + + def request(self, item, draft=False, gate=None): + body = {'model': 'lfm3-check', 'max_tokens': 24, 'temperature': item.get('temperature', 0), + 'seed': 1234, 'stream': True, 'stream_options': {'include_usage': True}, + 'ignore_eos': True, 'draft': draft} + if body['temperature']: + body.update(top_k=20, top_p=0.95) + chat = 'messages' in item + if chat: + body['messages'] = item['messages'] + body['chat_template_kwargs'] = {'enable_thinking': False} + else: + body['prompt'] = item['prompt'] + url = self.base + ('/v1/chat/completions' if chat else '/v1/completions') + request = urllib.request.Request(url, data=json.dumps(body).encode(), + headers={'Content-Type': 'application/json'}) + if gate: + gate.wait(timeout=30) + evidence, usage, runtime, text = Evidence(), {}, {}, [] + complete = False + try: + with urllib.request.urlopen(request, timeout=180) as response: + for raw in response: + line = raw.decode().strip() + if line == 'data: [DONE]': + complete = True + break + if not line.startswith('data:'): + continue + event = json.loads(line[5:]) + if 'error' in event: + raise RuntimeError(str(event['error'])) + usage = event.get('usage') or usage + runtime = event.get('tensorfold') or runtime + for choice in event.get('choices', []): + evidence.add(choice) + text.append(choice.get('text') or (choice.get('delta') or {}).get('content') or '') + result = evidence.finish(usage, runtime, complete) + result['text'] = ''.join(text) + if not complete: + result['error'] = 'incomplete_stream' + return result + except urllib.error.HTTPError as exc: + return {'error': exc.read(8192).decode(errors='replace'), 'status': exc.code, 'complete': False} + except Exception as exc: + return {'error': f'{type(exc).__name__}: {exc}', 'complete': False} + + + def disconnections(self): + with urllib.request.urlopen(self.base + '/metrics', timeout=5) as response: + text = response.read(1 << 20).decode() + match = re.search(r'^tensorfold:client_disconnections_total (\d+)$', text, re.MULTILINE) + if match is None: + raise RuntimeError('Missing server client-disconnection counter') + return int(match.group(1)) + + def cancel_stream(self, item): + before = self.disconnections() + body = {'model': 'lfm3-check', 'max_tokens': 512, 'temperature': 0, 'seed': 1234, + 'stream': True, 'ignore_eos': True, 'draft': True} + is_chat = 'messages' in item + if is_chat: + body['messages'] = item['messages'] + body['chat_template_kwargs'] = {'enable_thinking': False} + else: + body['prompt'] = item['prompt'] + endpoint = '/v1/chat/completions' if is_chat else '/v1/completions' + request = urllib.request.Request(self.base + endpoint, data=json.dumps(body).encode(), + headers={'Content-Type': 'application/json'}) + first_content, request_id, already_finished = None, None, False + with urllib.request.urlopen(request, timeout=30) as response: + for raw in response: + line = raw.decode().strip() + if line == 'data: [DONE]': + already_finished = True + break + if not line.startswith('data:'): + continue + event = json.loads(line[5:]) + if 'error' in event: + raise RuntimeError(str(event['error'])) + request_id = event.get('id') or request_id + for choice in event.get('choices', []): + delta = choice.get('delta') or {} + content = choice.get('text') or delta.get('content') or delta.get('reasoning_content') + already_finished = already_finished or choice.get('finish_reason') is not None + if content: + first_content = content + break + if first_content: + break + deadline = time.monotonic() + 20 + after = before + while time.monotonic() < deadline and after <= before and first_content and not already_finished: + after = self.disconnections() + if after <= before: + time.sleep(0.05) + return {'request_id': request_id, 'first_content': first_content, 'requested_tokens': 512, + 'closed_before_done': bool(first_content) and not already_finished, + 'server_disconnections_before': before, 'server_disconnections_after': after, + 'server_confirmed_cancel': after > before} + + +def concurrent_requests(server, items): + gate = threading.Barrier(len(items)) + with concurrent.futures.ThreadPoolExecutor(max_workers=len(items)) as executor: + futures = [executor.submit(server.request, item, True, gate) for item in items] + return [future.result() for future in futures] + + +def chat(text, images=()): + content = [image_url(image) for image in images] + [{'type': 'text', 'text': text}] + return {'messages': [{'role': 'user', 'content': content if images else text}]} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--binary', type=Path, required=True) + parser.add_argument('--model-dir', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--skip-images', action='store_true', help='Diagnostic omission, recorded as skipped') + parser.add_argument('--cancel', action='store_true', help='Also check real HTTP cancellation and recovery') + parser.add_argument('--cancel-only', action='store_true', help='Run only fresh baselines and cancellation checks') + parser.add_argument('--sampled', action='store_true', help='Also test seeded sampling') + args = parser.parse_args() + args.binary, args.model_dir = args.binary.resolve(), args.model_dir.resolve() + config = json.loads((args.model_dir / 'config.json').read_text()) + vision = config.get('model_type') == 'lfm3_vl' + receipt = {'model': args.model_dir.name, 'binary_sha256': hashlib.sha256(args.binary.read_bytes()).hexdigest(), + 'tests': [], 'logs': [], 'commands': [], 'speed_qualified': False} + + def check(name, passed, evidence=None, skip=False): + status = 'skipped' if skip else 'passed' if passed is True else 'failed' + receipt['tests'].append({'name': name, 'status': status, 'evidence': evidence}) + print(f'{status}: {name}', flush=True) + + code = {'prompt': 'Write a Python function that returns the larger of two integers.'} + prose = chat('Explain why the sky is blue in two sentences.') + story = ('The river flows past a quiet village. Children watch birds near the bridge. ' * 60) + long = {'prompt': story + '\nSummarize the story in one sentence.'} + edited = {'prompt': 'A mountain rises beside a busy town. ' + story[70:] + '\nSummarize in one sentence.'} + items = {'code': code, 'chat': prose, 'long_prefix': long, 'edited_prefix': edited} + if args.sampled: + items['sampled_code'] = dict(code, temperature=0.7) + items['sampled_chat'] = dict(prose, temperature=0.7) + if vision and not args.skip_images: + red, blue = solid_png(255, 0, 0), solid_png(0, 0, 255) + question = 'What single solid color fills this image? Reply with only the color name.' + items.update(red=chat(question, [red]), blue=chat(question, [blue])) + order = 'Name the solid color of the first image, then the second image. Reply with two color names in order.' + items.update(red_blue=chat(order, [red, blue]), blue_red=chat(order, [blue, red])) + elif vision: + check('images', None, 'Explicit --skip-images diagnostic run', skip=True) + if args.cancel_only: + items = {name: items[name] for name in ('code', 'chat', 'red') if name in items} + baselines = {} + log_directory = tempfile.mkdtemp(prefix='lfm3-serve-check-') + try: + with Server(args.binary, args.model_dir, 0, log_directory) as server: + receipt['logs'].append(server.log.name) + receipt['commands'].append(server.command) + for name, item in items.items(): + result = server.request(item) + baselines[name] = result + check(name + '/fresh_plain', result.get('complete') and bool(result.get('token_sha256')), result) + if 'long_prefix' in baselines: + check('long_prefix/length_over_256', (baselines['long_prefix'].get('prompt_tokens') or 0) > 256, + baselines['long_prefix'].get('prompt_tokens')) + with Server(args.binary, args.model_dir, 0.1, log_directory) as server: + receipt['logs'].append(server.log.name) + receipt['commands'].append(server.command) + if not args.cancel_only: + for name, item in items.items(): + for draft in (False, True): + result = server.request(item, draft=draft) + if name == 'edited_prefix' and not draft: + check('edited_prefix/no_stale_prefix_reuse', result.get('cached_tokens') == 0, result) + check(name + ('/drafted_equals_fresh' if draft else '/plain_equals_fresh'), + token_equal(result, baselines[name]), result) + if name == 'long_prefix': + warm = server.request(item, draft=False) + check('long_prefix/warm_reuses_and_matches_fresh', + (warm.get('cached_tokens') or 0) > 0 and token_equal(warm, baselines[name]) is True, warm) + if name == 'edited_prefix': + result = server.request(item) + check('edited_prefix/matches_cache_off', token_equal(result, baselines[name]), result) + for level in (1, 2, 4): + names = ['code', 'chat'] if not args.sampled else ['code', 'chat', 'sampled_code', 'sampled_chat'] + for name in names: + results = concurrent_requests(server, [items[name]] * level) + check(f'{name}/concurrent_{level}_equals_solo', + all(token_equal(result, baselines[name]) is True for result in results), results) + if vision and not args.skip_images: + for name, expected in [('red', ['red']), ('blue', ['blue']), + ('red_blue', ['red', 'blue']), ('blue_red', ['blue', 'red'])]: + found = re.findall(r'\b(?:red|blue)\b', baselines[name].get('text', '').lower()) + check(name + '/color_semantics', found[:len(expected)] == expected, + {'expected': expected, 'observed': found, 'text': baselines[name].get('text')}) + image_items = [items['red'], items['blue'], items['red_blue'], items['blue_red']] + results = concurrent_requests(server, image_items) + check('images/concurrent_equals_solos', all( + token_equal(result, baselines[name]) is True + for result, name in zip(results, ('red', 'blue', 'red_blue', 'blue_red')) + ), results) + corrupt = server.request(chat('Describe this image.', [b'\x89PNG\r\n\x1a\ninvalid'])) + check('images/corrupt_png_rejected', 400 <= corrupt.get('status', 0) < 500, corrupt) + if args.cancel or args.cancel_only: + for name in ('code', 'chat', 'red'): + if name not in items: + continue + cancelled = server.cancel_stream(items[name]) + check(name + '/cancel_server_confirmed', + cancelled['closed_before_done'] and cancelled['server_confirmed_cancel'], cancelled) + recovered = server.request(items[name], draft=False) + check(name + '/cancel_next_plain_equals_fresh', token_equal(recovered, baselines[name]), recovered) + recovered = server.request(items[name], draft=True) + check(name + '/cancel_next_drafted_equals_fresh', + token_equal(recovered, baselines[name]), recovered) + + except Exception as exc: + check('harness_execution', False, f'{type(exc).__name__}: {exc}') + receipt['counts'] = {status: sum(test['status'] == status for test in receipt['tests']) + for status in ('passed', 'failed', 'skipped')} + receipt['cancellation_checks'] = sum('/cancel_' in test['name'] for test in receipt['tests']) + args.output.write_text(json.dumps(receipt, indent=2) + '\n') + print(json.dumps(receipt['counts'])) + return int(receipt['counts']['failed'] > 0) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/lfm3/test_compare_logits.py b/tools/lfm3/test_compare_logits.py new file mode 100644 index 0000000000..19ddae8c54 --- /dev/null +++ b/tools/lfm3/test_compare_logits.py @@ -0,0 +1,44 @@ +"""Ensure fidelity receipts preserve real differences, ties and phase-independent row identities.""" +import unittest +try: + import numpy as np +except ImportError: + np = None +if np is not None: + from compare_logits import compare + + +@unittest.skipIf(np is None, 'NumPy is required for fidelity reports') +class CompareTests(unittest.TestCase): + def test_disjoint_contenders_and_both_margins_are_retained(self): + result = compare([[2, 1, 0], [0, 3, 1]], [[1, 2, 0], [0, 3, 1]], 7) + self.assertEqual(result['rows'], 2) + self.assertEqual(result['disjoint_maximum_sets'], 1) + row = result['differences'][0] + self.assertEqual((row['row'], row['native_top'], row['reference_top']), (7, 0, 1)) + self.assertEqual(row['native_margin'], 1) + self.assertEqual(row['reference_margin'], 1) + self.assertEqual(row['native_reference_contender_gap'], 1) + self.assertEqual(row['reference_native_contender_gap'], 1) + self.assertIsNone(result['fidelity_pass']) + + def test_overlapping_tie_is_reported_without_waiving_it(self): + result = compare([[1, 2, 0]], [[2, 2, 0]]) + self.assertEqual(result['top1_differences'], 1) + self.assertEqual(result['disjoint_maximum_sets'], 0) + self.assertEqual(result['differences'][0]['reference_margin'], 0) + self.assertIsNone(result['accepted_threshold']) + + def test_worst_rows_are_not_hidden_by_average(self): + result = compare([[0, 0], [9, 1]], [[0, 0], [1, 1]]) + self.assertEqual(result['worst_rows'][0]['row'], 1) + self.assertEqual(result['maximum_absolute_error'], 8) + + def test_invalid_geometry_and_nonfinite_inputs_are_refused(self): + for a, b in (([[1]], [[1]]), ([[1, 2]], [[1, 2], [1, 2]]), ([[np.nan, 1]], [[1, 2]])): + with self.assertRaises(ValueError): + compare(a, b) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/lfm3/test_fp32_graph_reference.py b/tools/lfm3/test_fp32_graph_reference.py new file mode 100644 index 0000000000..fc444d05e6 --- /dev/null +++ b/tools/lfm3/test_fp32_graph_reference.py @@ -0,0 +1,71 @@ +"""CPU graph precision modes, affine dequantization and BF16 midpoint boundaries.""" +import unittest +import numpy as np +from fp32_graph_reference import Graph + + +class PrecisionTests(unittest.TestCase): + def graph(self, mode): + graph = Graph.__new__(Graph) + graph.boundaries, graph.trace = mode, None + return graph + + def test_boundary_dtype_finite_and_mode_difference(self): + value = np.asarray([1.001, -1.001, 0, -0.0], dtype=np.float32) + pure, rounded = (self.graph(mode).cast(value) for mode in ('fp32', 'fp32-bf16')) + self.assertEqual(pure.dtype, np.float32) + self.assertEqual(rounded.dtype, np.float32) + self.assertTrue(np.isfinite(pure).all() and np.isfinite(rounded).all()) + np.testing.assert_array_equal(pure, value) + np.testing.assert_array_equal(rounded[:2], [1, -1]) + self.assertEqual(rounded.view(np.uint32)[3], 0x80000000) + + def test_bf16_rne_midpoints(self): + low = np.float32(1 + 2 ** -8) + high = np.float32(1 + 3 * 2 ** -8) + values = np.asarray([low, np.nextafter(low, np.float32(np.inf)), high, -low, -high]) + want = np.asarray([1, 1 + 2 ** -7, 1 + 2 ** -6, -1, -1 - 2 ** -6], dtype=np.float32) + np.testing.assert_array_equal(self.graph('fp32-bf16').cast(values), want) + + def test_fp32_product_sum_is_not_fp64_reduction(self): + x = np.asarray([[2 ** 24, 1]], dtype=np.float32) + y = x @ np.ones((2, 1), dtype=np.float32) + self.assertEqual(y.dtype, np.float32) + self.assertEqual(float(y[0, 0]), 2 ** 24) + self.assertEqual(float(x.astype(np.float64).sum()), 2 ** 24 + 1) + + def test_dequantization_uses_fp32_without_bf16_weight_rounding(self): + class Loader: + def quant(self, _): + return 8, 4 + + def tensor(self, name, _): + if name.endswith('.weight'): + return np.asarray([[0xffffffff]], dtype=np.uint32) + return np.asarray([[1.0078125 if name.endswith('.scales') else -128]], dtype=np.float64) + graph = self.graph('fp32-bf16') + graph.checkpoint = Loader() + weights = graph.weights('x', slice(None)) + self.assertEqual(weights.dtype, np.float32) + np.testing.assert_array_equal(weights, np.full((1, 4), np.float32(255 * 1.0078125 - 128))) + self.assertFalse(np.array_equal(weights, graph.cast(weights))) + + def test_attention_mask_and_dtype(self): + graph = self.graph('fp32') + q = np.zeros((1, 3, 2), dtype=np.float32) + v = np.asarray([[[1, 2], [3, 4], [99, 100]]], dtype=np.float32) + result = graph.attention_value(q, q, v, window=1) + self.assertEqual(result.dtype, np.float32) + np.testing.assert_array_equal(result, v) + np.testing.assert_array_equal(graph.attention_value(q, q, v)[0, 0], v[0, 0]) + + def test_swiglu_and_rms_stay_fp32(self): + graph = self.graph('fp32-bf16') + values = np.asarray([[1, -2, 3, -4]], dtype=np.float32) + for result in (graph.swiglu(values, values), graph.rms(values, np.ones(4, dtype=np.float32), 1e-5)): + self.assertEqual(result.dtype, np.float32) + self.assertTrue(np.isfinite(result).all()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/lfm3/test_fp64_reference.py b/tools/lfm3/test_fp64_reference.py new file mode 100644 index 0000000000..ba2696a6de --- /dev/null +++ b/tools/lfm3/test_fp64_reference.py @@ -0,0 +1,90 @@ +"""CPU reference checks for direct BF16 rounding and packed affine reconstruction.""" +import unittest + +try: + import numpy as np +except ImportError: + np = None + +if np is not None: + from fp64_reference import Oracle, bf, inject_image_rows + + +def bits(values): + return (np.asarray(bf(values), dtype=np.float32).view(np.uint32) >> 16).astype(np.uint16) + + +@unittest.skipIf(np is None, 'NumPy is required for the independent FP64 reference') +class ReferenceTests(unittest.TestCase): + def test_all_finite_values_and_adjacent_rounding_boundaries(self): + raw = np.arange(0x7f80, dtype=np.uint16) + values = (raw.astype(np.uint32) << 16).view(np.float32).astype(np.float64) + np.testing.assert_array_equal(bits(values), raw) + np.testing.assert_array_equal(bits(-values), raw | 0x8000) + middle = (values[:-1] + values[1:]) / 2 + even = np.where((raw[:-1] & 1) == 0, raw[:-1], raw[1:]) + np.testing.assert_array_equal(bits(middle), even) + np.testing.assert_array_equal(bits(-middle), even | 0x8000) + np.testing.assert_array_equal(bits(np.nextafter(middle, np.inf)), raw[1:]) + np.testing.assert_array_equal(bits(np.nextafter(middle, -np.inf)), raw[:-1]) + + def test_fp32_intermediate_would_double_round_the_fp64_result(self): + value = 1 + 2 ** -8 + 2 ** -30 + self.assertEqual(int(bits(value)), 0x3f81) + self.assertEqual(int(bits(np.float32(value))), 0x3f80) + self.assertEqual(int(bits(1 + 2 ** -8 - 2 ** -30)), 0x3f80) + + def test_subnormal_overflow_and_nonfinite_values(self): + self.assertEqual(int(bits(2 ** -134)), 0) + self.assertEqual(int(bits(np.nextafter(np.float64(2 ** -134), np.inf))), 1) + maximum = (np.uint32(0x7f7f) << np.uint32(16)).view(np.float32).astype(np.float64) + threshold = maximum + 2 ** 119 + self.assertEqual(int(bits(np.nextafter(threshold, -np.inf))), 0x7f7f) + self.assertEqual(int(bits(threshold)), 0x7f80) + self.assertEqual(int(bits(float('inf'))), 0x7f80) + self.assertEqual(int(bits(-float('inf'))), 0xff80) + self.assertEqual(int(bits(float('nan'))), 0x7fc0) + self.assertEqual(int(bits(-0.0)), 0x8000) + + def test_six_bit_codes_cross_word_boundaries_before_weight_rounding(self): + oracle = Oracle.__new__(Oracle) + oracle.config = {'quantization': {'bits': 6, 'group_size': 32}} + oracle.weight_mode = 'exact' + tensors = { + 'projection.weight': np.array([[0x440c2040, 0xa2481c61, 0x3ce34c2c, + 0x544d2450, 0xa6585d65, 0x7de75c6d]], dtype=np.uint32), + 'projection.scales': np.array([[1 / 256]], dtype=np.float64), + 'projection.biases': np.array([[8]], dtype=np.float64), + } + oracle.tensor = lambda name, rows=None: tensors[name] if rows is None else tensors[name][rows] + expected = 8 + np.arange(32, dtype=np.float64)[None] / 256 + np.testing.assert_array_equal(oracle.weights('projection', slice(0, 1)), expected) + oracle.weight_mode = 'bf16' + rounded = oracle.weights('projection', slice(0, 1)) + self.assertEqual(rounded[0, 1], 8) + self.assertNotEqual(rounded[0, 1], expected[0, 1]) + + def test_projected_image_rows_replace_only_matching_placeholders(self): + hidden = np.arange(12, dtype=np.float64).reshape(3, 4) + rows = np.array([[1 + 2 ** -8 + 2 ** -30, 2, 3, 4]]) + result = inject_image_rows(hidden, [10, 125030, 11], rows, 1) + np.testing.assert_array_equal(result[[0, 2]], hidden[[0, 2]]) + np.testing.assert_array_equal(hidden, np.arange(12).reshape(3, 4)) + self.assertEqual(result[1, 0], float(bf(rows[0, 0]))) + for tokens, value, start in (([10, 125030, 11], rows, 0), + ([10, 125030, 125030], rows, 1), + ([10, 125030, 11], rows[:, :3], 1), + ([10, 125030, 11], rows * np.nan, 1)): + with self.assertRaises(ValueError): + inject_image_rows(hidden, tokens, value, start) + + def test_embedding_override_does_not_change_other_projections(self): + oracle = Oracle.__new__(Oracle) + oracle.config = {'quantization': {'bits': 4, 'group_size': 32, + 'language_model.model.embed_tokens': {'bits': 8, 'group_size': 32}}} + self.assertEqual(oracle.quant('language_model.model.embed_tokens'), (8, 32)) + self.assertEqual(oracle.quant('language_model.model.layers.0.conv.in_proj'), (4, 32)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/lfm3/test_inspect_checkpoint.py b/tools/lfm3/test_inspect_checkpoint.py new file mode 100644 index 0000000000..d647542ade --- /dev/null +++ b/tools/lfm3/test_inspect_checkpoint.py @@ -0,0 +1,142 @@ +"""Host checks for bounded checkpoint metadata inspection.""" + +import io +import json +import struct +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import inspect_checkpoint as inspector + + +class Response(io.BytesIO): + def __init__(self, body, status, headers): + super().__init__(body) + self.status = status + self.headers = headers + self.read_calls = 0 + + def read(self, size=-1): + self.read_calls += 1 + return super().read(size) + + +class InspectorTests(unittest.TestCase): + def test_refuses_full_weight_response_before_reading(self): + response = Response(b'weight data', 200, {}) + with patch.object(inspector.urllib.request, 'urlopen', return_value=response): + with self.assertRaisesRegex(inspector.InspectionError, 'refused'): + inspector.Remote.fetch('https://example.org/model', 8, (0, 7)) + self.assertEqual(response.read_calls, 0) + + def test_rejects_wrong_range(self): + response = Response(b'12345678', 206, {'Content-Range': 'bytes 8-15/100'}) + with patch.object(inspector.urllib.request, 'urlopen', return_value=response): + with self.assertRaisesRegex(inspector.InspectionError, 'unexpected'): + inspector.Remote.fetch('https://example.org/model', 8, (0, 7)) + + def test_rejects_truncated_range(self): + response = Response(b'123', 206, {'Content-Range': 'bytes 0-7/100'}) + with patch.object(inspector.urllib.request, 'urlopen', return_value=response): + with self.assertRaisesRegex(inspector.InspectionError, 'Truncated'): + inspector.Remote.fetch('https://example.org/model', 8, (0, 7)) + + def test_accepts_exact_range(self): + response = Response(b'12345678', 206, {'Content-Range': 'bytes 0-7/100', 'Content-Length': '8'}) + with patch.object(inspector.urllib.request, 'urlopen', return_value=response): + self.assertEqual(inspector.Remote.fetch('https://example.org/model', 8, (0, 7)), b'12345678') + + def test_caps_non_range_metadata(self): + response = Response(b'123456789', 200, {}) + with patch.object(inspector.urllib.request, 'urlopen', return_value=response): + with self.assertRaisesRegex(inspector.InspectionError, 'body limit'): + inspector.Remote.fetch('https://example.org/config', 8) + + def test_rejects_oversize_header_and_unsafe_names(self): + with self.assertRaises(inspector.InspectionError): + inspector.header_size(struct.pack('> shift) & 0xffffffff for shift in range(0, 512 * bits, 32)] + for value in packed], dtype=np.uint32) + return Fixture() + + def test_all_three_packed_widths_match_independent_integer_sum(self): + for bits in (4, 6, 8): + result = affine_rows(self.oracle(bits), 'projection', np.ones((2, 512), dtype=np.float32)) + expected = 2 * sum(i % (1 << bits) for i in range(512)) - 512 + np.testing.assert_array_equal(result, np.full((2, 1), expected)) + + def test_noncontiguous_unpacked_columns_keep_their_identity(self): + values = np.arange(512, dtype=np.float32) % 64 - 32 + for bits in (4, 6, 8): + actual = affine_rows(self.oracle(bits, 3), 'projection', values[None]) + expected = [sum(float(values[i]) * ((i + 3 * c) % (1 << bits)) * (2 + c) + for i in range(512)) - float(values.sum()) for c in range(3)] + np.testing.assert_array_equal(actual, [expected]) + + def test_dense_fp32_rounding_occurs_before_final_storage(self): + values = np.zeros((1, 96), dtype=np.float32) + values[0, [0, 32, 64]] = [2 ** 24, 1, -(2 ** 24)] + weight = np.ones((1, 96), dtype=np.float32) + self.assertEqual((values.astype(np.float64) @ weight.astype(np.float64).T)[0, 0], 1) + self.assertEqual(float(dense_rows(values, weight)[0, 0]), 0) + + def test_dense_columns_and_input_guards(self): + values = (np.arange(64) % 8).astype(np.float32)[None] + weights = np.stack([np.ones(64), np.arange(64) % 4]).astype(np.float32)[::-1] + np.testing.assert_array_equal(dense_rows(values, weights), values @ weights.T) + with self.assertRaises(ValueError): + dense_rows(values[:, :-1], weights) + with self.assertRaises(ValueError): + dense_rows(np.full((1, 64), 1 + 2 ** -30), weights) + + def test_inputs_must_be_finite_bf16_rows_with_served_k_geometry(self): + for value in (np.ones((1, 511)), np.full((1, 512), 1.001), + np.full((1, 512), 1 + 2 ** -30), np.full((1, 512), np.inf)): + with self.assertRaises(ValueError): + affine_rows(self.oracle(4), 'projection', value) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/lfm3/test_text_attention_rounding.py b/tools/lfm3/test_text_attention_rounding.py new file mode 100644 index 0000000000..352e19cba6 --- /dev/null +++ b/tools/lfm3/test_text_attention_rounding.py @@ -0,0 +1,34 @@ +"""Check causal geometry and rotary identity for the optional FP32 replay.""" +import unittest +try: + import numpy as np +except ImportError: + np = None +if np is not None: + from text_attention_rounding import attention, rotary + + + +@unittest.skipIf(np is None, 'NumPy is required for rounding reference tests') +class TextAttentionRoundingTests(unittest.TestCase): + def test_causal_uniform_attention_uses_only_prefix(self): + q = np.zeros((2, 4, 128), dtype=np.float32) + v = np.broadcast_to(np.arange(1, 5, dtype=np.float32)[None, :, None], q.shape) + expected = np.broadcast_to(np.array([1, 1.5, 2, 2.5])[None, :, None], q.shape) + np.testing.assert_array_equal(attention(q, q, v, 0, 'numpy'), expected) + sliding = np.broadcast_to(np.array([1, 1.5, 2.5, 3.5])[None, :, None], q.shape) + np.testing.assert_array_equal(attention(q, q, v, 2, 'numpy'), sliding) + + def test_zero_position_is_identity_with_partial_rotary(self): + values = np.arange(128, dtype=np.float32).reshape(1, 1, 128) + spec = {'rope_theta': 10000, 'partial_rotary_factor': .5} + np.testing.assert_array_equal(rotary(values, spec, 'numpy'), values) + + def test_unimplemented_multi_partition_replay_is_rejected(self): + q = np.zeros((1, 1025, 128), dtype=np.float32) + with self.assertRaises(ValueError): + attention(q, q, q, 0, 'numpy') + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/lfm3/test_text_rounding_reference.py b/tools/lfm3/test_text_rounding_reference.py new file mode 100644 index 0000000000..af9e9b11d4 --- /dev/null +++ b/tools/lfm3/test_text_rounding_reference.py @@ -0,0 +1,53 @@ +"""The production reduction tree must not be replaced by an FP64 sum followed by one FP32 cast.""" +import unittest +try: + import numpy as np +except ImportError: + np = None +if np is not None: + from text_rounding_reference import norm_rows, sum32 + + +@unittest.skipIf(np is None, 'NumPy is required for the normalization rounding replay') +class TextRoundingTests(unittest.TestCase): + def test_positive_lane_sums_round_at_each_butterfly(self): + values = np.ones(32, dtype=np.float32) + values[0] = 2 ** 24 + self.assertEqual(sum32(values), 2 ** 24 + 30) + self.assertEqual(np.float32(values.astype(np.float64).sum()), 2 ** 24 + 32) + self.assertEqual(values[0], 2 ** 24) + + def test_hidden_and_head_norms_keep_rows_independent(self): + for width in (128, 2048): + values = np.ones((2, width), dtype=np.float32) + values[1] *= 2 + result = norm_rows(values, np.ones(width), 1e-5) + np.testing.assert_array_equal(result, np.ones_like(values)) + np.testing.assert_array_equal(values[1], np.full(width, 2)) + + def test_bad_geometry_or_non_bf16_inputs_are_refused(self): + for value, gain in ((np.ones((1, 64)), np.ones(64)), + (np.full((1, 128), 1 + 2 ** -30), np.ones(128)), + (np.ones((1, 128)), np.ones(127))): + with self.assertRaises(ValueError): + norm_rows(value, gain, 1e-5) + +@unittest.skipIf(np is None, 'NumPy is required for rounding reference tests') +class MetalFlushBoundaryTests(unittest.TestCase): + def test_named_metal_mode_flushes_fp32_subnormal_gain_product(self): + from unittest.mock import patch + values = np.ones((1, 128)) + values[0, 0] = .5 + gain = np.ones(128) + gain[0] = 2 ** -126 + ideal = norm_rows(values, gain, 1e-5) + self.assertGreater(ideal[0, 0], 0) + with patch('elementary_reference.elementary', + side_effect=lambda x, operation, backend: (1 / np.sqrt(x.astype(np.float64))).astype(np.float32)): + actual = norm_rows(values, gain, 1e-5, 'metal') + self.assertEqual(actual[0, 0], 0) + np.testing.assert_array_equal(actual[:, 1:], ideal[:, 1:]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/lfm3/test_vision_fp64_reference.py b/tools/lfm3/test_vision_fp64_reference.py new file mode 100644 index 0000000000..998099571b --- /dev/null +++ b/tools/lfm3/test_vision_fp64_reference.py @@ -0,0 +1,33 @@ +"""Verify vision oracle rounding and reject malformed patch grids before evaluating weights.""" +import unittest +from types import SimpleNamespace +try: + import numpy as np +except ImportError: + np = None +if np is not None: + from vision_fp64_reference import VisionOracle + + +@unittest.skipIf(np is None, 'NumPy is required for the FP64 vision reference') +class VisionOracleTests(unittest.TestCase): + def test_dense_rounds_the_fp64_sum_directly_to_bf16(self): + oracle = VisionOracle.__new__(VisionOracle) + oracle.loader = SimpleNamespace(map={}) + oracle.weight = lambda name: np.ones((1, 3), dtype=np.float64) + actual = oracle.dense(np.array([[1, 2 ** -8, 2 ** -30]], dtype=np.float64), 'projection') + self.assertEqual(actual[0, 0], 1 + 2 ** -7) + self.assertEqual(float(np.float32(1 + 2 ** -8 + 2 ** -30)), 1 + 2 ** -8) + + def test_grid_and_patch_layout_are_validated_before_weight_access(self): + oracle = VisionOracle.__new__(VisionOracle) + for height, width, patches in ((0, 2, np.zeros((0, 1536))), + (3, 2, np.zeros((6, 1536))), + (2, 2, np.zeros((4, 768))), + (2, 2, np.full((4, 1536), np.nan))): + with self.assertRaises(ValueError): + oracle.run(patches, height, width) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/lfm3/test_vision_rounding_reference.py b/tools/lfm3/test_vision_rounding_reference.py new file mode 100644 index 0000000000..e569e3f95e --- /dev/null +++ b/tools/lfm3/test_vision_rounding_reference.py @@ -0,0 +1,65 @@ +"""Check vision replay indexing and intermediate FP32 rounding independently.""" +import unittest +try: + import numpy as np +except ImportError: + np = None +if np is not None: + from elementary_reference import fma, swiglu + from vision_rounding_reference import rms_rows, convolution, rotary, attention, layer_norm, gelu + + + +@unittest.skipIf(np is None, 'NumPy is required for rounding reference tests') +class VisionRoundingTests(unittest.TestCase): + def test_fma_retains_product_residual_before_round(self): + a, b = np.float32(1 + 2 ** -23), np.float32(1 - 2 ** -23) + self.assertEqual(float(fma(a, b, -1)), -2 ** -46) + self.assertEqual(float(np.float32(a * b) - np.float32(1)), 0) + + def test_vision_rms_keeps_strided_row_channels_and_gain(self): + values = np.ones((2, 1024)) + gain = np.ones(1024) + gain[513] = -2 + actual = rms_rows(values, gain, 1e-5) + np.testing.assert_array_equal(actual, np.tile(gain, (2, 1))) + with self.assertRaises(ValueError): + rms_rows(np.ones((1, 128)), np.ones(128), 1e-5) + + def test_spatial_convolution_zero_padding_and_channel_axis(self): + grid = np.arange(24).reshape(2, 3, 4) + kernel = np.zeros((4, 3, 3)) + kernel[:, 1, 1] = np.arange(1, 5) + np.testing.assert_array_equal(convolution(grid, kernel), grid * np.arange(1, 5)) + kernel[:] = 0 + kernel[:, 0, 1] = 1 + expected = np.zeros_like(grid) + expected[1] = grid[0] + np.testing.assert_array_equal(convolution(grid, kernel), expected) + + def test_zero_position_rotary_is_identity(self): + values = np.arange(128, dtype=np.float32).reshape(2, 1, 64) + np.testing.assert_array_equal(rotary(values, np.zeros((2, 2))), values) + + def test_attention_constant_visible_values_are_preserved(self): + q = np.zeros((2, 5, 64), dtype=np.float32) + v = np.broadcast_to(np.arange(64, dtype=np.float32), q.shape) + np.testing.assert_array_equal(attention(q, q, v), v) + with self.assertRaises(ValueError): + attention(q[..., :32], q, v) + + def test_projector_constant_row_and_gelu_origin(self): + np.testing.assert_array_equal(layer_norm(np.ones((2, 1024)), np.ones(1024), + np.zeros(1024), 1e-6), np.zeros((2, 1024))) + np.testing.assert_array_equal(gelu(np.array([0], dtype=np.float32)), [0]) + + def test_swiglu_rounds_silu_before_up_product(self): + from fp64_reference import bf + gate = np.array([0, 1, -1], dtype=np.float32) + up = np.array([3, 2, 2], dtype=np.float32) + silu = bf(gate / (np.float32(1) + np.exp(-gate))) + np.testing.assert_array_equal(swiglu(gate, up), bf(silu * up)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/lfm3/text_attention_rounding.py b/tools/lfm3/text_attention_rounding.py new file mode 100644 index 0000000000..667b423a2c --- /dev/null +++ b/tools/lfm3/text_attention_rounding.py @@ -0,0 +1,57 @@ +"""Independent bounded FP32 replay of decoder rotary and one-partition attention.""" +import math +import numpy as np +from fp64_reference import bf +from elementary_reference import elementary, fma +from text_rounding_reference import sum32 + + +def rotary(values, spec, backend): + values = np.asarray(values, dtype=np.float32) + pairs = int(values.shape[-1] * spec.get('partial_rotary_factor', 1)) // 2 + frequency = np.arange(64, dtype=np.float32) + exponent = -frequency / np.float32(64) * np.float32(math.log2(spec['rope_theta'])) + frequency = elementary(exponent, 'exp2', backend) + frequency[pairs:] = 0 + angle = np.arange(len(values), dtype=np.float32)[:, None] * frequency + cosine = bf(elementary(angle, 'cos', backend)).astype(np.float32)[:, None] + sine = bf(elementary(angle, 'sin', backend)).astype(np.float32)[:, None] + a, b = values[..., :64], values[..., 64:] + return bf(np.concatenate((fma(a, cosine, -(b * sine)), fma(b, cosine, a * sine)), axis=-1)) + + +def attention(q, k, v, window, backend): + q, k, v = (np.asarray(value, dtype=np.float32) for value in (q, k, v)) + heads, rows, width = q.shape + if width != 128 or rows > 1024 or k.shape != q.shape or v.shape != q.shape: + raise ValueError('Named text attention replay requires at most 1024 equally expanded 128-channel rows') + output = np.empty_like(q) + for first in range(0, rows, 16): + query = q[:, first:first + 16] + products = np.zeros((heads, len(query[0]), rows, 32), dtype=np.float32) + for part in range(4): + products = fma(query[:, :, None, part * 32:(part + 1) * 32], + k[:, None, :, part * 32:(part + 1) * 32], products) + scores = sum32(products) * np.float32(1 / math.sqrt(128)) + distance = np.arange(first, first + len(query[0]))[:, None] - np.arange(rows)[None] + allowed = distance >= 0 + if window: + allowed &= distance < window + scores = np.where(allowed[None], scores, np.float32(-np.inf)) + maxima = np.maximum.accumulate(scores, axis=-1) + previous = np.concatenate((np.full((*scores.shape[:2], 1), -np.inf, np.float32), maxima[..., :-1]), axis=-1) + with np.errstate(invalid='ignore'): + alpha = elementary(previous - maxima, 'exp', backend) + probability = elementary(scores - maxima, 'exp', backend) + alpha = np.where(np.isneginf(previous), np.float32(0), alpha) + probability = np.where(np.isneginf(scores), np.float32(0), probability) + total = np.zeros(scores.shape[:2], dtype=np.float32) + result = np.zeros((*total.shape, 128), dtype=np.float32) + for key in range(rows): + active = allowed[None, :, key] + next_total = fma(total, alpha[..., key], probability[..., key]) + next_result = fma(result, alpha[..., key, None], probability[..., key, None] * v[:, key, None]) + total = np.where(active, next_total, total) + result = np.where(active[..., None], next_result, result) + output[:, first:first + len(query[0])] = result / total[..., None] + return bf(output) diff --git a/tools/lfm3/text_rounding_reference.py b/tools/lfm3/text_rounding_reference.py new file mode 100644 index 0000000000..5322e0c764 --- /dev/null +++ b/tools/lfm3/text_rounding_reference.py @@ -0,0 +1,52 @@ +"""Independent replay of the text RMS normalization's prescribed FP32 partial and lane reductions.""" +import numpy as np +from fp64_reference import bf + + +def sum32(values): + result = np.asarray(values, dtype=np.float32) + if result.shape[-1] != 32: + raise ValueError('Expected exactly 32 reduction lanes') + for distance in (1, 2, 4, 8, 16): + result = result + result[..., np.arange(32) ^ distance] + return result[..., 0] + + +def norm_rows(values, gain, epsilon, elementary_backend=None): + original, gain = np.asarray(values), np.asarray(gain) + if original.ndim == 0 or not np.isfinite(original).all() or not np.array_equal(original, bf(original)): + raise ValueError('Expected finite BF16 normalization inputs') + values = np.asarray(original, dtype=np.float32) + width = values.shape[-1] + if gain.shape != (width,) or not np.isfinite(gain).all() or not np.array_equal(gain, bf(gain)): + raise ValueError('Expected one finite BF16 normalization gain per column') + if not np.isfinite(epsilon) or epsilon <= 0: + raise ValueError('Expected positive finite normalization epsilon') + if width == 2048: + partial = values.reshape(*values.shape[:-1], 256, 8) + square = np.zeros(partial.shape[:-1], dtype=np.float32) + for at in range(8): + square = (partial[..., at].astype(np.float64) ** 2 + square).astype(np.float32) + groups = sum32(square.reshape(*values.shape[:-1], 8, 32)) + padded = np.zeros((*values.shape[:-1], 32), dtype=np.float32) + padded[..., :8] = groups + total = sum32(padded) + elif width == 128: + partial = values.reshape(*values.shape[:-1], 4, 32) + square = np.zeros((*values.shape[:-1], 32), dtype=np.float32) + for at in range(4): + square = (partial[..., at, :].astype(np.float64) ** 2 + square).astype(np.float32) + total = sum32(square) + else: + raise ValueError('Expected a text hidden row or attention head') + variance = total / np.float32(width) + np.float32(epsilon) + inverse = (1 / np.sqrt(variance.astype(np.float64))).astype(np.float32)[..., None] + if elementary_backend is not None: + from elementary_reference import elementary + inverse = elementary(variance, 'rsqrt', elementary_backend)[..., None] + first = bf(values * inverse).astype(np.float32) + result = first * np.asarray(gain, dtype=np.float32) + if elementary_backend == 'metal': + # Metal's FP32 arithmetic flushes subnormal products, before the BF16 store. + result = np.where(np.abs(result) < np.finfo(np.float32).tiny, np.copysign(np.float32(0), result), result) + return bf(result) diff --git a/tools/lfm3/vision_attention_rounding.py b/tools/lfm3/vision_attention_rounding.py new file mode 100644 index 0000000000..5db27c723c --- /dev/null +++ b/tools/lfm3/vision_attention_rounding.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Diagnose production elementary rounding; never declare an independent FP64 fidelity pass.""" +import argparse +import hashlib +import json +from pathlib import Path +import numpy as np +from elementary_reference import fma +from text_rounding_reference import sum32 + + +def read(directory, name, rows, width): + words = np.fromfile(directory / (name + '.bf16'), dtype=' 16384: + raise ValueError('Expected a valid even patch grid') + if patches.shape != (height * width, 1536) or not np.isfinite(patches).all(): + raise ValueError('Expected finite merge-ordered patches with 1536 elements each') + coordinates = np.array([(h * 2 + a, w * 2 + b) for h in range(height // 2) + for w in range(width // 2) for a in range(2) for b in range(2)]) + hidden = self.dense(bf(patches), 'vision_tower.patchifier') + self.take('patchifier', hidden) + hidden = self.rms(hidden, 'vision_tower.embedding_norm') + self.take('embedding_norm', hidden) + for index, kind in enumerate(self.vision['layer_types']): + prefix = f'vision_tower.layers.{index}.' + x = self.rms(hidden, prefix + 'operator_norm') + if index == 0: + self.take('first_operator_norm', x) + result = (self.convolution(x, prefix, height, width) if kind == 'conv' + else self.attention(x, prefix, coordinates)) + if index == 0: + self.take('first_operator', result) + hidden = bf(hidden + result) + if index == 0: + self.take('first_residual', hidden) + x = self.rms(hidden, prefix + 'ffn_norm') + if index == 0: + self.take('first_ffn_norm', x) + gate, up = self.dense(x, prefix + 'feed_forward.w1'), self.dense(x, prefix + 'feed_forward.w3') + if index == 0: + self.take('first_ffn_gate', gate) + self.take('first_ffn_up', up) + if getattr(self, 'operator_rounding', 'fp64') == 'metal-fp32': + from elementary_reference import swiglu + act = swiglu(gate, up, self.elementary_backend) + else: + act = bf(bf(gate / (1 + np.exp(-gate))) * up) + if index == 0: + self.take('first_ffn_act', act) + down = self.dense(act, prefix + 'feed_forward.w2') + if index == 0: + self.take('first_ffn_down', down) + hidden = bf(hidden + down) + self.take(f'layer_{index:02d}', hidden) + print(f'vision_layer_{index:02d}', flush=True) + prefix = 'multi_modal_projector.' + centered = hidden - hidden.mean(axis=-1, keepdims=True) + inverse = 1 / np.sqrt(np.mean(centered * centered, axis=-1, keepdims=True) + + self.config['projector_layer_norm_eps']) + normalized = bf(centered * inverse * self.weight(prefix + 'layer_norm.weight') + + self.weight(prefix + 'layer_norm.bias')) + if getattr(self, 'operator_rounding', 'fp64') == 'metal-fp32': + from vision_rounding_reference import layer_norm + normalized = layer_norm(hidden, self.weight(prefix + 'layer_norm.weight'), + self.weight(prefix + 'layer_norm.bias'), self.config['projector_layer_norm_eps'], + self.elementary_backend) + self.take('projector_norm', normalized) + merged = normalized.reshape(-1, self.vision['hidden_size'] * self.config['downsample_factor'] ** 2) + first = self.dense(merged, prefix + 'linear_1') + erf = np.fromiter((math.erf(v / math.sqrt(2)) for v in first.flat), dtype=np.float64).reshape(first.shape) + activated = bf(0.5 * first * (1 + erf)) + if getattr(self, 'operator_rounding', 'fp64') == 'metal-fp32': + from vision_rounding_reference import gelu + activated = gelu(first, self.elementary_backend) + self.take('projector_gelu', activated) + return self.dense(activated, prefix + 'linear_2') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--model-dir', type=Path, required=True) + parser.add_argument('--patches', type=Path, required=True) + parser.add_argument('--grid-h', type=int, required=True) + parser.add_argument('--grid-w', type=int, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--trace-dir', type=Path) + parser.add_argument('--elementary-backend', choices=('numpy', 'metal'), default='numpy') + parser.add_argument('--operator-rounding', choices=('fp64', 'metal-fp32'), default='fp64') + parser.add_argument('--dense-rounding', choices=('fp64', 'metal-row'), default='fp64') + args = parser.parse_args() + def write(path, values): + path.write_bytes((np.asarray(values, dtype=np.float32).view(np.uint32) >> 16).astype('